Renamed nlTok.tokType to nlTok.tType

This commit is contained in:
Emile Clark-Boman 2025-06-17 19:10:42 +10:00
parent 12c200b13f
commit edf164df90
3 changed files with 20 additions and 21 deletions

View file

@ -1 +0,0 @@
rename nlTok.tokType to nlTok.tType

View file

@ -26,7 +26,7 @@ type
HASH, # # Number Sign (Hashtag)
nlTok = object
tokType*: nlTokType
tType*: nlTokType
lit*: string
line*: Natural
startPos*: Natural
@ -34,23 +34,23 @@ type
# Generates an "empty" nlTok with only a startPos,
# all other fields are expected to be filled out later.
# NOTE: tokType initialised to nlTokType.NUL
# NOTE: tType initialised to nlTokType.NUL
# NOTE: lit initialised to empty string
# NOTE: all other fields are uninitialised
proc emptyTok(startPos: int): nlTok =
result = nlTok(
tokType: nlTokType.NONE,
tType: nlTokType.NONE,
lit: "",
startPos: Natural startPos,
)
# Checks if an nlTok has nlTokType.NONE
proc isTokUntyped(tokType: nlTokType): bool =
result = (tokType == nlTokType.NONE)
proc isTokUntyped(tType: nlTokType): bool =
result = (tType == nlTokType.NONE)
# Checks if an nlTok has nlTokType.TERM
proc isTokTerm(tokType: nlTokType): bool =
result = (tokType == nlTokType.TERM)
proc isTokTerm(tType: nlTokType): bool =
result = (tType == nlTokType.TERM)
# This method is only used to convert null
# terminator nlToks into line-feed ones.
@ -59,7 +59,7 @@ proc isTokTerm(tokType: nlTokType): bool =
# NOTE: strings in a useful but annoying way
proc tokTermToLineFeed(tok: nlTok): nlTok =
result = nlTok(
tokType: nlTokType.LNFD,
tType: nlTokType.LNFD,
lit: tok.lit,
line: tok.line,
startPos: tok.startPos,

View file

@ -8,7 +8,7 @@ type
build: nlTok # the current token we're building
# Resets the build token to an "empty" nlTok where
# only tokType, lit, and startPos are initialised.
# only tType, lit, and startPos are initialised.
proc resetBuild(tokStream: var nlTokStream) =
tokStream.build = emptyTok(tokStream.lstream.pos)
@ -18,7 +18,7 @@ proc resetBuild(tokStream: var nlTokStream) =
proc finishBuild(tokStream: var nlTokStream) =
# if we've reached \0 terminator then forge the start
# and end positions to point OUTSIDE the line
let endPos = if isTokTerm(tokStream.build.tokType):
let endPos = if isTokTerm(tokStream.build.tType):
inc tokStream.build.startPos;
tokStream.build.startPos
else: Natural tokStream.lstream.pos
@ -36,30 +36,30 @@ proc flushBuild(tokStream: var nlTokStream): nlTok =
# This indicates that the build token should inherit
# the nlTokType of the nlLStream's next character.
proc isUntypedBuild(tokStream: nlTokStream): bool =
result = isTokUntyped(tokStream.build.tokType)
result = isTokUntyped(tokStream.build.tType)
# Check whether an nlTokType is "compatible" with
# the build token. flushBuild() should be called
# when an incompatible token is discovered.
proc isCompatibleBuild(tokStream: nlTokStream, tokType: nlTokType): bool =
result = (tokType == tokStream.build.tokType)
proc isCompatibleBuild(tokStream: nlTokStream, tType: nlTokType): bool =
result = (tType == tokStream.build.tType)
# Add a character to the nlTokStream's build token.
# Returns a bool indicating if a new nlTok has been built
# or not. flushBuild should then be called.
proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] =
let tokType = getTokType(c)
let tType = getTokType(c)
# check whether build token should inherit type
if isUntypedBuild(tokStream):
tokStream.build.tokType = tokType
tokStream.build.tType = tType
# check character and build token compatability
elif not isCompatibleBuild(tokStream, tokType):
elif not isCompatibleBuild(tokStream, tType):
# return flushed build token, and reset
result = some(flushBuild(tokStream))
# new build token is untyped so inherit type
tokStream.build.tokType = tokType
tokStream.build.tType = tType
# check if \0 terminator reached
elif isTokTerm(tokStream.build.tokType):
elif isTokTerm(tokStream.build.tType):
# return immediately to avoid concatinating '\0'
return some(flushBuild(tokStream))
# else return none to indicate no build was completed
@ -69,7 +69,7 @@ proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] =
tokStream.build.lit.add(c)
# Generates and returns the next token in the stream,
# result.tokType == nlTokType.NTERM implies line ended
# result.tType == nlTokType.NTERM implies line ended
proc nextTok(tokStream: var nlTokStream): nlTok =
# try progress to next char, receives none option on failure
for optchar in iterChars(tokStream.lstream):
@ -96,7 +96,7 @@ iterator toks*(tokStream: var nlTokStream): nlTok =
tok = nextTok(tokStream)
# \0 terminator means the line ended OR the file
# has ended, so always yield a line-feed just in case
if isTokTerm(tok.tokType):
if isTokTerm(tok.tType):
yield tokTermToLineFeed(tok)
break
yield tok