Tokenisation now accessible via the nlTokStream interface

nlTokStream relies on the functionality of nlLStream

src/nlx.nim

@@ -0,0 +1,13 @@
+import os
+import noether/lex
+
+when isMainModule:
+  echo "Noether Lang - Extras"
+
+  if paramCount() > 0:
+    let filename = paramStr(1)
+    var tokStream = newTokStream(filename, isFile=true)
+    for tok in toks(tokStream):
+      echo tok
+  else:
+    echo "usage: nlx filename"