From 07a9bda9ba017f1d60eed2e719b55264bdc90dcd Mon Sep 17 00:00:00 2001
From: Emile Clark-Boman <eclarkboman@gmail.com>
Date: Thu, 19 Jun 2025 09:11:49 +1000
Subject: [PATCH] Once again fixed EOL handling...

---
 src/noether/lexer/tokbuilder.nim | 51 +++++++++++---------------------
 src/noether/lexer/tokstream.nim  |  4 +--
 2 files changed, 19 insertions(+), 36 deletions(-)

diff --git a/src/noether/lexer/tokbuilder.nim b/src/noether/lexer/tokbuilder.nim
index 357841a..0d2f212 100644
--- a/src/noether/lexer/tokbuilder.nim
+++ b/src/noether/lexer/tokbuilder.nim
@@ -54,70 +54,55 @@ proc finishBuild(builder: var nlTokBuilder) =
 
 # Finish, return, and reset the build token
 proc flushBuild(builder: var nlTokBuilder): nlTok = 
-  echo "Flush @", builder.pos
   finishBuild(builder)
   result = builder.tok
   resetBuild(builder)
 
-# Is the build token "compatible" with the current char?
-# NOTE: flushBuild() is called if incompatible
-proc isCompatibleBuild(builder: nlTokBuilder): bool =
-  result = (builder.cTKind == builder.tok.kind)
+# Is the build token "compatible" with the current char? (if not then flushbuild)
+# NOTE: This implicitly handles Windows CRLF, Unix LF, & Mac OS CR compatability 
+# NOTE: since atEOL => '\n', but '\r' and '\n' are both tkEOL so they both flush.
+proc isIncompatibleBuild(builder: nlTokBuilder): bool =
+  result = (builder.cTKind != builder.tok.kind or builder.atEOL())
 
 # Inherit the build token's type from current char
 proc inherit(builder: var nlTokBuilder) = 
   builder.tok.kind = builder.cTKind
 
 # Add a character to the nlTokBuilder's build token.
-# Flushes and returns the build token if "fully built",
-# and a boolean indicating whether the nlTokBuilder can progress.
-proc appendBuild(builder: var nlTokBuilder, flushed: var Option[nlTok]): bool =    
+# Flushes and returns the build token if finished.
+proc appendBuild(builder: var nlTokBuilder): Option[nlTok] =    
   # untyped build tokens inherit type immediately
   if builder.tok.isUntyped():
     builder.inherit()
   
-  # check if EOF reached
-  # if builder.atEOL():
-  #     echo "EOL DETECT 1"
-  #     result = false # DO NOT PROGRESS
-  #     flushed = some(flushBuild(builder))
   # check character and build token compatability
-  if not isCompatibleBuild(builder):
+  if isIncompatibleBuild(builder):
       # flush old build token, the new one inherits type
-      flushed = some(flushBuild(builder))
+      result = some(flushBuild(builder))
       builder.inherit()
-      result = true # can progress
   else:
-    flushed = none(nlTok)
-    result = true # can progress
+    result = none(nlTok)
 
 #[ ========================================== ]
  | nlTokBuilder Char Stream Reading Interface ]
  ]#
       
-# Read the next char in the stream without
-# checking whether it is safe to do so
-proc forceReadChar(builder: var nlTokBuilder) {.inline.} = 
-  echo "read"
-  inc builder.pos
+# Read the next char in the stream
+# NOTE: readChar raises IOError on error, returns \0 on EOF
+proc readChar*(builder: var nlTokBuilder): bool =
+  if builder.atEOL():
+    inc builder.lineNum
+  # sets builder.char to '\0' if EOF
   builder.char = builder.stream.readChar()
   builder.cTKind = getTokKind(builder.char)
   builder.line.add(builder.char)
-
-# Read the next char in the stream
-# NOTE: readChar raises IOError on error, returns \0 on EOF
-proc readChar(builder: var nlTokBuilder): bool =
-  if builder.atEOL():
-    echo "EOL DETECT 2"
-    inc builder.lineNum
-  # sets builder.char to '\0' if EOF
-  builder.forceReadChar()
+  inc builder.pos
   result = builder.atEOF()
 
 # Read until EOL and return the current line
 # NOTE: Does NOT update the builder's state (unsafe)
 # NOTE: ONLY call if a lex/parse error needs displaying
-proc unsafeGetLine(builder: var nlTokBuilder): string =
+proc unsafeGetLine*(builder: var nlTokBuilder): string =
   while not builder.atEOL() and builder.readChar():
     discard
   result = builder.line
diff --git a/src/noether/lexer/tokstream.nim b/src/noether/lexer/tokstream.nim
index 309e9bb..e64f777 100644
--- a/src/noether/lexer/tokstream.nim
+++ b/src/noether/lexer/tokstream.nim
@@ -31,12 +31,10 @@ proc progress*(stream: var nlTokStream): bool =
   if stream.isClosed:
     return false
   while true:
-    # echo "\nProgressing..."
-    var flushedTok: Option[nlTok]
     let 
       atEOF = stream.builder.readChar()
+      flushedTok = stream.builder.appendBuild()
       newTokBuilt = flushedTok.isSome
-    discard stream.builder.appendBuild(flushedTok)
     echo flushedTok
     echo "atEOF: ", atEOF, "\nnewTokBuilt: ", newTokBuilt
     # canProgress & EOF reached => no more tokens to build :)