From dff40e6b275a340d03b21a1f522b6d94eb59d468 Mon Sep 17 00:00:00 2001
From: Emile Clark-Boman <eclarkboman@gmail.com>
Date: Tue, 17 Jun 2025 13:08:21 +1000
Subject: [PATCH] Added support for various nlTokTypes and added multiple demos

---
 README                     |  1 +
 lang/NOTES                 |  3 +++
 lang/demo/loop-space.no    |  2 ++
 lang/demo/loop-tab-crlf.no |  2 ++
 lang/demo/loop-tab.no      |  2 ++
 lang/demo/string.no        |  3 +++
 src/nlx.nim                |  2 +-
 src/noether/tokens.nim     | 41 +++++++++++++++++++++++++++-----------
 8 files changed, 43 insertions(+), 13 deletions(-)
 create mode 100644 lang/NOTES
 create mode 100644 lang/demo/loop-space.no
 create mode 100644 lang/demo/loop-tab-crlf.no
 create mode 100644 lang/demo/loop-tab.no
 create mode 100644 lang/demo/string.no

diff --git a/README b/README
index 67cc69b..cdc7436 100644
--- a/README
+++ b/README
@@ -7,6 +7,7 @@ A personal little MATLAB alternative I suppose :)
 In future I'd love to start using a developed TUI library like [textual](https://github.com/Textualize/textual?tab=readme-ov-file),
 but for now everything is just a custom ANSI wrapper thing.
 
+Noether can compile to C or Nim via their respective backends.
 
 
 NOTES:
diff --git a/lang/NOTES b/lang/NOTES
new file mode 100644
index 0000000..0e8e5c9
--- /dev/null
+++ b/lang/NOTES
@@ -0,0 +1,3 @@
+Todo:
+- [ ] Not currently sure how the lexer will interpret non-latin characters (make sure it handles all unicode)
+- [ ] The lexer currently only handles a limited number of escape codes / whitespace characters
diff --git a/lang/demo/loop-space.no b/lang/demo/loop-space.no
new file mode 100644
index 0000000..f652755
--- /dev/null
+++ b/lang/demo/loop-space.no
@@ -0,0 +1,2 @@
+loop:
+    print("Goodbye world!")
diff --git a/lang/demo/loop-tab-crlf.no b/lang/demo/loop-tab-crlf.no
new file mode 100644
index 0000000..57a689a
--- /dev/null
+++ b/lang/demo/loop-tab-crlf.no
@@ -0,0 +1,2 @@
+loop:
+	print("Goodbye world!")
diff --git a/lang/demo/loop-tab.no b/lang/demo/loop-tab.no
new file mode 100644
index 0000000..41aefec
--- /dev/null
+++ b/lang/demo/loop-tab.no
@@ -0,0 +1,2 @@
+loop:
+	print("Goodbye world!")
diff --git a/lang/demo/string.no b/lang/demo/string.no
new file mode 100644
index 0000000..f7f816d
--- /dev/null
+++ b/lang/demo/string.no
@@ -0,0 +1,3 @@
+"Goodbye World!"
+'a' 
+'abc' # ERROR
diff --git a/src/nlx.nim b/src/nlx.nim
index a32c0b8..b79cd2e 100644
--- a/src/nlx.nim
+++ b/src/nlx.nim
@@ -2,7 +2,7 @@ import os
 import noether/lex
 
 when isMainModule:
-  echo "Noether Lang - Extras"
+  echo "Noether Lang Extras v0.1.0 - nlx"
 
   if paramCount() > 0:
     let filename = paramStr(1)
diff --git a/src/noether/tokens.nim b/src/noether/tokens.nim
index a4a4bcf..6e18151 100644
--- a/src/noether/tokens.nim
+++ b/src/noether/tokens.nim
@@ -10,16 +10,19 @@ type
     TERM, # String \0 terminator
     WORD, # Alphanumeric token
     SYMB, # Symbolic token
-    LNFD, # Line-Feed
-    WTSP, # Whitespace
-    LPAR, # (
-    RPAR, # )
-    LBRA, # {
-    RBRA, # }
-    LSQB, # [
-    RSQB, # ]
-    LANB, # <
-    RANB, # >
+    LNFD, # \r \n Line-Feed
+    WTSP, # ' ' \t Whitespace
+    LPAR, # ( Left Parenthesis
+    RPAR, # ) Right Parenthesis
+    LBRA, # { Left Brace
+    RBRA, # } Right Brace
+    LSQB, # [ Left Square Bracket
+    RSQB, # ] Right Square Bracket
+    # LANB, # < Left Angle Bracket
+    # RANB, # > Right Angle Bracket
+    SQUO, # ' Single Quotation Marking
+    DQUO, # " Double Quotation Marking
+    GRVA, # ` Grave Accent
         
   nlTok = object
     tokType*: nlTokType
@@ -67,13 +70,27 @@ proc getTokType(c: char): nlTokType =
   case c:
   of '\0':
     result = nlTokType.TERM
-  of '\n':
+  of '\r', '\n':
     result = nlTokType.LNFD
-  of ' ':
+  of ' ', '\t':
     result = nlTokType.WTSP
   of '(':
     result = nlTokType.LPAR
   of ')':
     result = nlTokType.RPAR
+  of '{':
+    result = nlTokType.LBRA
+  of '}':
+    result = nlTokType.RBRA
+  of '[':
+    result = nlTokType.LSQB
+  of ']':
+    result = nlTokType.RSQB
+  of '\'':
+    result = nlTokType.SQUO
+  of '\"':
+    result = nlTokType.DQUO
+  of '`':
+    result = nlTokType.GRVA
   else:
     result = nlTokType.WORD