From 90ca138904ff3ec23fe0348a84ac881fe81c37af Mon Sep 17 00:00:00 2001
From: Emile Clark-Boman <eclarkboman@gmail.com>
Date: Wed, 18 Jun 2025 02:35:51 +1000
Subject: [PATCH] Fixed build tokens not unwrapping when both EOL and EOF occur

---
 src/nlx.nim                       |  3 ++-
 src/noether/lexer/tokbuilding.nim |  1 +
 src/noether/lexer/tokstream.nim   | 19 ++++++++++++++-----
 src/noether/parser/parser.nim     | 17 +++++++++++++++--
 4 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/src/nlx.nim b/src/nlx.nim
index da182d0..2e590ac 100644
--- a/src/nlx.nim
+++ b/src/nlx.nim
@@ -8,7 +8,8 @@ when isMainModule:
   if paramCount() > 0:
     let filename = paramStr(1)
     var tokStream = newTokStream(filename, isFile=true)
-
+    
+    # DumpTok
     var tok: nlTok
     while tokStream.nextTok(tok):
       echo tok
diff --git a/src/noether/lexer/tokbuilding.nim b/src/noether/lexer/tokbuilding.nim
index c77ae6f..043ac71 100644
--- a/src/noether/lexer/tokbuilding.nim
+++ b/src/noether/lexer/tokbuilding.nim
@@ -6,6 +6,7 @@ type
   nlTokStream = object
     lstream: nlLStream
     build: nlTok # the build token
+    closed: bool # EOF + all tokens built
 
 # Generates an EOL token for the nlTokStream's state
 proc EOLTok*(tokStream: nlTokStream): nlTok = 
diff --git a/src/noether/lexer/tokstream.nim b/src/noether/lexer/tokstream.nim
index 6358d1d..98f92b7 100644
--- a/src/noether/lexer/tokstream.nim
+++ b/src/noether/lexer/tokstream.nim
@@ -4,6 +4,7 @@ include tokbuilding
 proc newTokStream*(content: string, isFile: bool = false): nlTokStream =
   result = nlTokStream(
     lstream: newLStream(content, isFile=isFile),
+    closed: false,
   )
   # 1. initialise an empty build token 
   # 2. progress to the first line
@@ -27,12 +28,20 @@ proc progChar(tokStream: var nlTokStream): bool =
 # NOTE: progBuild adds lstream's current char to the build token
 # NOTE: progChar progresses to lstream's next char
 proc nextTok*(tokStream: var nlTokStream, tok: var nlTok): bool =
+  # Return prematurely if already closed
+  if tokStream.closed:
+    return false
   while true:
     var buildTok: Option[nlTok]
-    let canProgress = tokStream.progBuild(buildTok)
-    # canProgress & progression failed => EOF reached
-    if canProgress and not tokStream.progChar():
-      return false
-    elif buildTok.isSome:
+    let 
+      canProgress = tokStream.progBuild(buildTok)
+      tokBuilt = buildTok.isSome
+    # canProgress & EOF reached => no more tokens to build :)
+    # NOTE: reachedEOF and not canProgress => more tokens unwrapping
+    if tokBuilt:
       tok = buildTok.get()
+    if canProgress and not tokStream.progChar():
+      tokStream.closed = true
+      return tokBuilt
+    elif tokBuilt:
       return true
diff --git a/src/noether/parser/parser.nim b/src/noether/parser/parser.nim
index 42888c8..b1937c9 100644
--- a/src/noether/parser/parser.nim
+++ b/src/noether/parser/parser.nim
@@ -1,7 +1,20 @@
+import ../lexer/tokstream
+
+# Greed will consume anything except a punishment
+proc greed(tokStream: nlTokStream, toks: var seq[nlTok], punish: str) =
+  
+
+proc parse_strl(tokStream: nlTokStream): nlNode =
+  
+
 # Attempt to form an nlAST from a nlTokStream
-proc arborise(tokStream: nlTokStream): nlNode = 
-  for tok in toks(tokStream):
+proc parse(tokStream: nlTokStream): nlNode = 
+  var tok: nlTok
+  while true:
     case tok.tokType:
     of nlTokType.DQUO:
       # Attempt to parse string literal
       parse_strl()
+
+    if not tokStream.nextTok(tok):
+      break