85 lines
3.3 KiB
Nim
85 lines
3.3 KiB
Nim
include lstream
|
|
|
|
type
|
|
# Provides a stream-like interface for lexing nlToks
|
|
# Internally reliant on the functionality of nlLStream
|
|
nlTokStream = object
|
|
lstream: nlLStream
|
|
build: nlTok # the build token
|
|
closed: bool # EOF + all tokens built
|
|
|
|
# Generates an EOL token for the nlTokStream's state
|
|
proc EOLTok*(tokStream: nlTokStream): nlTok =
|
|
result = nlTok(
|
|
tType: nlTokType.EOL,
|
|
lit: "\0",
|
|
lineNum: Natural tokStream.lstream.lineNum,
|
|
startPos: Natural tokStream.lstream.pos,
|
|
endPos: Natural tokStream.lstream.pos,
|
|
)
|
|
|
|
# Resets the build token to an "empty" nlTok
|
|
proc resetBuild(tokStream: var nlTokStream) =
|
|
tokStream.build = emptyTok(tokStream.lstream.pos)
|
|
|
|
# Completes a token generated by emptyTok()
|
|
# based on the nlTokStream's nlLStream's
|
|
# current line and character positions
|
|
proc finishBuild(ts: var nlTokStream) =
|
|
ts.build.lineNum = Natural ts.lstream.lineNum
|
|
ts.build.endPos = Natural ts.lstream.pos
|
|
ts.build.lit = ts.lstream.line[ts.build.startPos ..< ts.build.endPos]
|
|
|
|
# Returns the nlTokStream's build token and
|
|
# empties the build token's contents.
|
|
proc flushBuild(tokStream: var nlTokStream): nlTok =
|
|
finishBuild(tokStream)
|
|
result = tokStream.build
|
|
resetBuild(tokStream)
|
|
|
|
# Returns whether the build token has a set type yet.
|
|
# This indicates that the build token should inherit
|
|
# the nlTokType of the nlLStream's next character.
|
|
proc isUntypedBuild(tokStream: nlTokStream): bool =
|
|
result = isTokUntyped(tokStream.build.tType)
|
|
|
|
# Check whether an nlTokType is "compatible" with the build token.
|
|
# NOTE: flushBuild() should be called when an incompatible token is discovered.
|
|
proc isCompatibleBuild(tokStream: nlTokStream, tType: nlTokType): bool =
|
|
result = (tType == tokStream.build.tType)
|
|
|
|
# Add a character to the nlTokStream's build token.
|
|
# Flushes and returns the build token if "fully built",
|
|
# and a boolean indicating whether the nlTokStream can progress.
|
|
proc progBuild(tokStream: var nlTokStream, buildTok: var Option[nlTok]): bool =
|
|
# the "pos > EOL" invalid state is used intentionally
|
|
# to indicate all tokens have been built, and return EOL Token
|
|
if tokStream.lstream.outOfBounds():
|
|
buildTok = some(EOLTok(tokStream))
|
|
return true # can progress once more
|
|
|
|
let tType = getTokType(tokStream.lstream.currChar())
|
|
# untyped build tokens must inherited immediately
|
|
if isUntypedBuild(tokStream):
|
|
tokStream.build.tType = tType
|
|
|
|
# check if EOL reached
|
|
if tokStream.lstream.atEOL():
|
|
# flush old build token, the new one can be left untyped
|
|
let compatible = isCompatibleBuild(tokStream, tType)
|
|
result = false # DO NOT PROGRESS
|
|
if compatible:
|
|
# force the lstream into an invalid state by progressing beyond EOL
|
|
# we can then detect this state on the next progBuild and return
|
|
# an EOL character (very unsafe implementation but it works well)
|
|
tokStream.lstream.forceProgChar()
|
|
buildTok = some(flushBuild(tokStream))
|
|
# check character and build token compatability
|
|
elif not isCompatibleBuild(tokStream, tType):
|
|
# flush old build token, the new one inherits type
|
|
buildTok = some(flushBuild(tokStream))
|
|
tokStream.build.tType = tType
|
|
result = true # can progress
|
|
else:
|
|
buildTok = none(nlTok)
|
|
result = true # can progress
|