Garbage collection *so to speak*

src/nlx.nim

@@ -1,6 +1,6 @@
 import os
 import noether/lib/io
-import noether/lexer/tokbuilder
+import noether/lexer/lex
 # import noether/parser/parser
 
 {.hint: "Don't forget to drink more water (^_^)".}

src/noether/lexer/tokstream.nim

@@ -1,51 +0,0 @@
-include tokbuilder
-
-type
-  # Provides a stream-like interface for lexing.
-  # Implemented as a wrapper for nlTokBuilder.
-  nlTokStream* = object
-    builder: nlTokBuilder
-    tok*: nlTok # the current token
-    isClosed: bool # EOF + all tokens built
-
-# Initialises a new nlTokStream on a string or file
-proc newTokStream*(stream: var Stream): nlTokStream =
-  result = nlTokStream(
-    builder: newBuilder(stream),
-    tok: emptyTok(0),
-    isClosed: false,
-  )
-  
-# Expose a subset of the nlTokBuilder interface
-proc line*(stream: nlTokStream): string =
-  result = stream.builder.line
-proc atEOL*(stream: nlTokStream): bool = 
-  result = stream.builder.atEOL()
-    
-# Generates and progress the next token in the nlTokStream.
-# via repeatedly calling progressBuild() and progressChar().
-# Returns a boolean indicating whether EOF has been reached.
-# NOTE: access the new token via `stream.tok`
-proc progress*(stream: var nlTokStream): bool =
-  # Return prematurely if already closed
-  if stream.isClosed:
-    return false
-  while true:
-    let 
-      atEOF = stream.builder.readChar()
-      flushedTok = stream.builder.appendBuild()
-      newTokBuilt = flushedTok.isSome
-    echo flushedTok
-    echo "atEOF: ", atEOF, "\nnewTokBuilt: ", newTokBuilt
-    # canProgress & EOF reached => no more tokens to build :)
-    # NOTE: reachedEOF and not canProgress => more tokens unwrapping
-    if newTokBuilt:
-      # return the finished build token, and save it as the current token
-      stream.tok = flushedTok.get()
-    # if canProgress and atEOF:
-    if atEOF:
-      if newTokBuilt:
-        stream.isClosed = true
-      return newTokBuilt
-    elif newTokBuilt:
-      return true