Renamed nlTok.tokType to nlTok.tType
This commit is contained in:
parent
12c200b13f
commit
edf164df90
3 changed files with 20 additions and 21 deletions
|
|
@ -1 +0,0 @@
|
|||
rename nlTok.tokType to nlTok.tType
|
||||
|
|
@ -26,7 +26,7 @@ type
|
|||
HASH, # # Number Sign (Hashtag)
|
||||
|
||||
nlTok = object
|
||||
tokType*: nlTokType
|
||||
tType*: nlTokType
|
||||
lit*: string
|
||||
line*: Natural
|
||||
startPos*: Natural
|
||||
|
|
@ -34,23 +34,23 @@ type
|
|||
|
||||
# Generates an "empty" nlTok with only a startPos,
|
||||
# all other fields are expected to be filled out later.
|
||||
# NOTE: tokType initialised to nlTokType.NUL
|
||||
# NOTE: tType initialised to nlTokType.NUL
|
||||
# NOTE: lit initialised to empty string
|
||||
# NOTE: all other fields are uninitialised
|
||||
proc emptyTok(startPos: int): nlTok =
|
||||
result = nlTok(
|
||||
tokType: nlTokType.NONE,
|
||||
tType: nlTokType.NONE,
|
||||
lit: "",
|
||||
startPos: Natural startPos,
|
||||
)
|
||||
|
||||
# Checks if an nlTok has nlTokType.NONE
|
||||
proc isTokUntyped(tokType: nlTokType): bool =
|
||||
result = (tokType == nlTokType.NONE)
|
||||
proc isTokUntyped(tType: nlTokType): bool =
|
||||
result = (tType == nlTokType.NONE)
|
||||
|
||||
# Checks if an nlTok has nlTokType.TERM
|
||||
proc isTokTerm(tokType: nlTokType): bool =
|
||||
result = (tokType == nlTokType.TERM)
|
||||
proc isTokTerm(tType: nlTokType): bool =
|
||||
result = (tType == nlTokType.TERM)
|
||||
|
||||
# This method is only used to convert null
|
||||
# terminator nlToks into line-feed ones.
|
||||
|
|
@ -59,7 +59,7 @@ proc isTokTerm(tokType: nlTokType): bool =
|
|||
# NOTE: strings in a useful but annoying way
|
||||
proc tokTermToLineFeed(tok: nlTok): nlTok =
|
||||
result = nlTok(
|
||||
tokType: nlTokType.LNFD,
|
||||
tType: nlTokType.LNFD,
|
||||
lit: tok.lit,
|
||||
line: tok.line,
|
||||
startPos: tok.startPos,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ type
|
|||
build: nlTok # the current token we're building
|
||||
|
||||
# Resets the build token to an "empty" nlTok where
|
||||
# only tokType, lit, and startPos are initialised.
|
||||
# only tType, lit, and startPos are initialised.
|
||||
proc resetBuild(tokStream: var nlTokStream) =
|
||||
tokStream.build = emptyTok(tokStream.lstream.pos)
|
||||
|
||||
|
|
@ -18,7 +18,7 @@ proc resetBuild(tokStream: var nlTokStream) =
|
|||
proc finishBuild(tokStream: var nlTokStream) =
|
||||
# if we've reached \0 terminator then forge the start
|
||||
# and end positions to point OUTSIDE the line
|
||||
let endPos = if isTokTerm(tokStream.build.tokType):
|
||||
let endPos = if isTokTerm(tokStream.build.tType):
|
||||
inc tokStream.build.startPos;
|
||||
tokStream.build.startPos
|
||||
else: Natural tokStream.lstream.pos
|
||||
|
|
@ -36,30 +36,30 @@ proc flushBuild(tokStream: var nlTokStream): nlTok =
|
|||
# This indicates that the build token should inherit
|
||||
# the nlTokType of the nlLStream's next character.
|
||||
proc isUntypedBuild(tokStream: nlTokStream): bool =
|
||||
result = isTokUntyped(tokStream.build.tokType)
|
||||
result = isTokUntyped(tokStream.build.tType)
|
||||
|
||||
# Check whether an nlTokType is "compatible" with
|
||||
# the build token. flushBuild() should be called
|
||||
# when an incompatible token is discovered.
|
||||
proc isCompatibleBuild(tokStream: nlTokStream, tokType: nlTokType): bool =
|
||||
result = (tokType == tokStream.build.tokType)
|
||||
proc isCompatibleBuild(tokStream: nlTokStream, tType: nlTokType): bool =
|
||||
result = (tType == tokStream.build.tType)
|
||||
|
||||
# Add a character to the nlTokStream's build token.
|
||||
# Returns a bool indicating if a new nlTok has been built
|
||||
# or not. flushBuild should then be called.
|
||||
proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] =
|
||||
let tokType = getTokType(c)
|
||||
let tType = getTokType(c)
|
||||
# check whether build token should inherit type
|
||||
if isUntypedBuild(tokStream):
|
||||
tokStream.build.tokType = tokType
|
||||
tokStream.build.tType = tType
|
||||
# check character and build token compatability
|
||||
elif not isCompatibleBuild(tokStream, tokType):
|
||||
elif not isCompatibleBuild(tokStream, tType):
|
||||
# return flushed build token, and reset
|
||||
result = some(flushBuild(tokStream))
|
||||
# new build token is untyped so inherit type
|
||||
tokStream.build.tokType = tokType
|
||||
tokStream.build.tType = tType
|
||||
# check if \0 terminator reached
|
||||
elif isTokTerm(tokStream.build.tokType):
|
||||
elif isTokTerm(tokStream.build.tType):
|
||||
# return immediately to avoid concatinating '\0'
|
||||
return some(flushBuild(tokStream))
|
||||
# else return none to indicate no build was completed
|
||||
|
|
@ -69,7 +69,7 @@ proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] =
|
|||
tokStream.build.lit.add(c)
|
||||
|
||||
# Generates and returns the next token in the stream,
|
||||
# result.tokType == nlTokType.NTERM implies line ended
|
||||
# result.tType == nlTokType.NTERM implies line ended
|
||||
proc nextTok(tokStream: var nlTokStream): nlTok =
|
||||
# try progress to next char, receives none option on failure
|
||||
for optchar in iterChars(tokStream.lstream):
|
||||
|
|
@ -96,7 +96,7 @@ iterator toks*(tokStream: var nlTokStream): nlTok =
|
|||
tok = nextTok(tokStream)
|
||||
# \0 terminator means the line ended OR the file
|
||||
# has ended, so always yield a line-feed just in case
|
||||
if isTokTerm(tok.tokType):
|
||||
if isTokTerm(tok.tType):
|
||||
yield tokTermToLineFeed(tok)
|
||||
break
|
||||
yield tok
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue