Fixed build tokens not unwrapping when both EOL and EOF occur

src/nlx.nim

@@ -8,7 +8,8 @@ when isMainModule:
   if paramCount() > 0:
     let filename = paramStr(1)
     var tokStream = newTokStream(filename, isFile=true)
-
+    
+    # DumpTok
     var tok: nlTok
     while tokStream.nextTok(tok):
       echo tok

src/noether/lexer/tokbuilding.nim

@@ -6,6 +6,7 @@ type
   nlTokStream = object
     lstream: nlLStream
     build: nlTok # the build token
+    closed: bool # EOF + all tokens built
 
 # Generates an EOL token for the nlTokStream's state
 proc EOLTok*(tokStream: nlTokStream): nlTok = 

src/noether/lexer/tokstream.nim

@@ -4,6 +4,7 @@ include tokbuilding
 proc newTokStream*(content: string, isFile: bool = false): nlTokStream =
   result = nlTokStream(
     lstream: newLStream(content, isFile=isFile),
+    closed: false,
   )
   # 1. initialise an empty build token 
   # 2. progress to the first line
@@ -27,12 +28,20 @@ proc progChar(tokStream: var nlTokStream): bool =
 # NOTE: progBuild adds lstream's current char to the build token
 # NOTE: progChar progresses to lstream's next char
 proc nextTok*(tokStream: var nlTokStream, tok: var nlTok): bool =
+  # Return prematurely if already closed
+  if tokStream.closed:
+    return false
   while true:
     var buildTok: Option[nlTok]
-    let canProgress = tokStream.progBuild(buildTok)
+    let 
-    # canProgress & progression failed => EOF reached
+      canProgress = tokStream.progBuild(buildTok)
-    if canProgress and not tokStream.progChar():
+      tokBuilt = buildTok.isSome
-      return false
+    # canProgress & EOF reached => no more tokens to build :)
-    elif buildTok.isSome:
+    # NOTE: reachedEOF and not canProgress => more tokens unwrapping
+    if tokBuilt:
       tok = buildTok.get()
+    if canProgress and not tokStream.progChar():
+      tokStream.closed = true
+      return tokBuilt
+    elif tokBuilt:
       return true

src/noether/parser/parser.nim

@@ -1,7 +1,20 @@
+import ../lexer/tokstream
+
+# Greed will consume anything except a punishment
+proc greed(tokStream: nlTokStream, toks: var seq[nlTok], punish: str) =
+  
+
+proc parse_strl(tokStream: nlTokStream): nlNode =
+  
+
 # Attempt to form an nlAST from a nlTokStream
-proc arborise(tokStream: nlTokStream): nlNode = 
+proc parse(tokStream: nlTokStream): nlNode = 
-  for tok in toks(tokStream):
+  var tok: nlTok
+  while true:
     case tok.tokType:
     of nlTokType.DQUO:
       # Attempt to parse string literal
       parse_strl()
+
+    if not tokStream.nextTok(tok):
+      break