Renamed nlTok.tokType to nlTok.tType

This commit is contained in:
Emile Clark-Boman 2025-06-17 19:10:42 +10:00
parent 12c200b13f
commit edf164df90
3 changed files with 20 additions and 21 deletions

View file

@ -1 +0,0 @@
rename nlTok.tokType to nlTok.tType

View file

@ -26,7 +26,7 @@ type
HASH, # # Number Sign (Hashtag) HASH, # # Number Sign (Hashtag)
nlTok = object nlTok = object
tokType*: nlTokType tType*: nlTokType
lit*: string lit*: string
line*: Natural line*: Natural
startPos*: Natural startPos*: Natural
@ -34,23 +34,23 @@ type
# Generates an "empty" nlTok with only a startPos, # Generates an "empty" nlTok with only a startPos,
# all other fields are expected to be filled out later. # all other fields are expected to be filled out later.
# NOTE: tokType initialised to nlTokType.NUL # NOTE: tType initialised to nlTokType.NUL
# NOTE: lit initialised to empty string # NOTE: lit initialised to empty string
# NOTE: all other fields are uninitialised # NOTE: all other fields are uninitialised
proc emptyTok(startPos: int): nlTok = proc emptyTok(startPos: int): nlTok =
result = nlTok( result = nlTok(
tokType: nlTokType.NONE, tType: nlTokType.NONE,
lit: "", lit: "",
startPos: Natural startPos, startPos: Natural startPos,
) )
# Checks if an nlTok has nlTokType.NONE # Checks if an nlTok has nlTokType.NONE
proc isTokUntyped(tokType: nlTokType): bool = proc isTokUntyped(tType: nlTokType): bool =
result = (tokType == nlTokType.NONE) result = (tType == nlTokType.NONE)
# Checks if an nlTok has nlTokType.TERM # Checks if an nlTok has nlTokType.TERM
proc isTokTerm(tokType: nlTokType): bool = proc isTokTerm(tType: nlTokType): bool =
result = (tokType == nlTokType.TERM) result = (tType == nlTokType.TERM)
# This method is only used to convert null # This method is only used to convert null
# terminator nlToks into line-feed ones. # terminator nlToks into line-feed ones.
@ -59,7 +59,7 @@ proc isTokTerm(tokType: nlTokType): bool =
# NOTE: strings in a useful but annoying way # NOTE: strings in a useful but annoying way
proc tokTermToLineFeed(tok: nlTok): nlTok = proc tokTermToLineFeed(tok: nlTok): nlTok =
result = nlTok( result = nlTok(
tokType: nlTokType.LNFD, tType: nlTokType.LNFD,
lit: tok.lit, lit: tok.lit,
line: tok.line, line: tok.line,
startPos: tok.startPos, startPos: tok.startPos,

View file

@ -8,7 +8,7 @@ type
build: nlTok # the current token we're building build: nlTok # the current token we're building
# Resets the build token to an "empty" nlTok where # Resets the build token to an "empty" nlTok where
# only tokType, lit, and startPos are initialised. # only tType, lit, and startPos are initialised.
proc resetBuild(tokStream: var nlTokStream) = proc resetBuild(tokStream: var nlTokStream) =
tokStream.build = emptyTok(tokStream.lstream.pos) tokStream.build = emptyTok(tokStream.lstream.pos)
@ -18,7 +18,7 @@ proc resetBuild(tokStream: var nlTokStream) =
proc finishBuild(tokStream: var nlTokStream) = proc finishBuild(tokStream: var nlTokStream) =
# if we've reached \0 terminator then forge the start # if we've reached \0 terminator then forge the start
# and end positions to point OUTSIDE the line # and end positions to point OUTSIDE the line
let endPos = if isTokTerm(tokStream.build.tokType): let endPos = if isTokTerm(tokStream.build.tType):
inc tokStream.build.startPos; inc tokStream.build.startPos;
tokStream.build.startPos tokStream.build.startPos
else: Natural tokStream.lstream.pos else: Natural tokStream.lstream.pos
@ -36,30 +36,30 @@ proc flushBuild(tokStream: var nlTokStream): nlTok =
# This indicates that the build token should inherit # This indicates that the build token should inherit
# the nlTokType of the nlLStream's next character. # the nlTokType of the nlLStream's next character.
proc isUntypedBuild(tokStream: nlTokStream): bool = proc isUntypedBuild(tokStream: nlTokStream): bool =
result = isTokUntyped(tokStream.build.tokType) result = isTokUntyped(tokStream.build.tType)
# Check whether an nlTokType is "compatible" with # Check whether an nlTokType is "compatible" with
# the build token. flushBuild() should be called # the build token. flushBuild() should be called
# when an incompatible token is discovered. # when an incompatible token is discovered.
proc isCompatibleBuild(tokStream: nlTokStream, tokType: nlTokType): bool = proc isCompatibleBuild(tokStream: nlTokStream, tType: nlTokType): bool =
result = (tokType == tokStream.build.tokType) result = (tType == tokStream.build.tType)
# Add a character to the nlTokStream's build token. # Add a character to the nlTokStream's build token.
# Returns a bool indicating if a new nlTok has been built # Returns a bool indicating if a new nlTok has been built
# or not. flushBuild should then be called. # or not. flushBuild should then be called.
proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] = proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] =
let tokType = getTokType(c) let tType = getTokType(c)
# check whether build token should inherit type # check whether build token should inherit type
if isUntypedBuild(tokStream): if isUntypedBuild(tokStream):
tokStream.build.tokType = tokType tokStream.build.tType = tType
# check character and build token compatability # check character and build token compatability
elif not isCompatibleBuild(tokStream, tokType): elif not isCompatibleBuild(tokStream, tType):
# return flushed build token, and reset # return flushed build token, and reset
result = some(flushBuild(tokStream)) result = some(flushBuild(tokStream))
# new build token is untyped so inherit type # new build token is untyped so inherit type
tokStream.build.tokType = tokType tokStream.build.tType = tType
# check if \0 terminator reached # check if \0 terminator reached
elif isTokTerm(tokStream.build.tokType): elif isTokTerm(tokStream.build.tType):
# return immediately to avoid concatinating '\0' # return immediately to avoid concatinating '\0'
return some(flushBuild(tokStream)) return some(flushBuild(tokStream))
# else return none to indicate no build was completed # else return none to indicate no build was completed
@ -69,7 +69,7 @@ proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] =
tokStream.build.lit.add(c) tokStream.build.lit.add(c)
# Generates and returns the next token in the stream, # Generates and returns the next token in the stream,
# result.tokType == nlTokType.NTERM implies line ended # result.tType == nlTokType.NTERM implies line ended
proc nextTok(tokStream: var nlTokStream): nlTok = proc nextTok(tokStream: var nlTokStream): nlTok =
# try progress to next char, receives none option on failure # try progress to next char, receives none option on failure
for optchar in iterChars(tokStream.lstream): for optchar in iterChars(tokStream.lstream):
@ -96,7 +96,7 @@ iterator toks*(tokStream: var nlTokStream): nlTok =
tok = nextTok(tokStream) tok = nextTok(tokStream)
# \0 terminator means the line ended OR the file # \0 terminator means the line ended OR the file
# has ended, so always yield a line-feed just in case # has ended, so always yield a line-feed just in case
if isTokTerm(tok.tokType): if isTokTerm(tok.tType):
yield tokTermToLineFeed(tok) yield tokTermToLineFeed(tok)
break break
yield tok yield tok