Renamed nlTok.tokType to nlTok.tType
This commit is contained in:
parent
12c200b13f
commit
edf164df90
3 changed files with 20 additions and 21 deletions
|
|
@ -1 +0,0 @@
|
||||||
rename nlTok.tokType to nlTok.tType
|
|
||||||
|
|
@ -26,7 +26,7 @@ type
|
||||||
HASH, # # Number Sign (Hashtag)
|
HASH, # # Number Sign (Hashtag)
|
||||||
|
|
||||||
nlTok = object
|
nlTok = object
|
||||||
tokType*: nlTokType
|
tType*: nlTokType
|
||||||
lit*: string
|
lit*: string
|
||||||
line*: Natural
|
line*: Natural
|
||||||
startPos*: Natural
|
startPos*: Natural
|
||||||
|
|
@ -34,23 +34,23 @@ type
|
||||||
|
|
||||||
# Generates an "empty" nlTok with only a startPos,
|
# Generates an "empty" nlTok with only a startPos,
|
||||||
# all other fields are expected to be filled out later.
|
# all other fields are expected to be filled out later.
|
||||||
# NOTE: tokType initialised to nlTokType.NUL
|
# NOTE: tType initialised to nlTokType.NUL
|
||||||
# NOTE: lit initialised to empty string
|
# NOTE: lit initialised to empty string
|
||||||
# NOTE: all other fields are uninitialised
|
# NOTE: all other fields are uninitialised
|
||||||
proc emptyTok(startPos: int): nlTok =
|
proc emptyTok(startPos: int): nlTok =
|
||||||
result = nlTok(
|
result = nlTok(
|
||||||
tokType: nlTokType.NONE,
|
tType: nlTokType.NONE,
|
||||||
lit: "",
|
lit: "",
|
||||||
startPos: Natural startPos,
|
startPos: Natural startPos,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Checks if an nlTok has nlTokType.NONE
|
# Checks if an nlTok has nlTokType.NONE
|
||||||
proc isTokUntyped(tokType: nlTokType): bool =
|
proc isTokUntyped(tType: nlTokType): bool =
|
||||||
result = (tokType == nlTokType.NONE)
|
result = (tType == nlTokType.NONE)
|
||||||
|
|
||||||
# Checks if an nlTok has nlTokType.TERM
|
# Checks if an nlTok has nlTokType.TERM
|
||||||
proc isTokTerm(tokType: nlTokType): bool =
|
proc isTokTerm(tType: nlTokType): bool =
|
||||||
result = (tokType == nlTokType.TERM)
|
result = (tType == nlTokType.TERM)
|
||||||
|
|
||||||
# This method is only used to convert null
|
# This method is only used to convert null
|
||||||
# terminator nlToks into line-feed ones.
|
# terminator nlToks into line-feed ones.
|
||||||
|
|
@ -59,7 +59,7 @@ proc isTokTerm(tokType: nlTokType): bool =
|
||||||
# NOTE: strings in a useful but annoying way
|
# NOTE: strings in a useful but annoying way
|
||||||
proc tokTermToLineFeed(tok: nlTok): nlTok =
|
proc tokTermToLineFeed(tok: nlTok): nlTok =
|
||||||
result = nlTok(
|
result = nlTok(
|
||||||
tokType: nlTokType.LNFD,
|
tType: nlTokType.LNFD,
|
||||||
lit: tok.lit,
|
lit: tok.lit,
|
||||||
line: tok.line,
|
line: tok.line,
|
||||||
startPos: tok.startPos,
|
startPos: tok.startPos,
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ type
|
||||||
build: nlTok # the current token we're building
|
build: nlTok # the current token we're building
|
||||||
|
|
||||||
# Resets the build token to an "empty" nlTok where
|
# Resets the build token to an "empty" nlTok where
|
||||||
# only tokType, lit, and startPos are initialised.
|
# only tType, lit, and startPos are initialised.
|
||||||
proc resetBuild(tokStream: var nlTokStream) =
|
proc resetBuild(tokStream: var nlTokStream) =
|
||||||
tokStream.build = emptyTok(tokStream.lstream.pos)
|
tokStream.build = emptyTok(tokStream.lstream.pos)
|
||||||
|
|
||||||
|
|
@ -18,7 +18,7 @@ proc resetBuild(tokStream: var nlTokStream) =
|
||||||
proc finishBuild(tokStream: var nlTokStream) =
|
proc finishBuild(tokStream: var nlTokStream) =
|
||||||
# if we've reached \0 terminator then forge the start
|
# if we've reached \0 terminator then forge the start
|
||||||
# and end positions to point OUTSIDE the line
|
# and end positions to point OUTSIDE the line
|
||||||
let endPos = if isTokTerm(tokStream.build.tokType):
|
let endPos = if isTokTerm(tokStream.build.tType):
|
||||||
inc tokStream.build.startPos;
|
inc tokStream.build.startPos;
|
||||||
tokStream.build.startPos
|
tokStream.build.startPos
|
||||||
else: Natural tokStream.lstream.pos
|
else: Natural tokStream.lstream.pos
|
||||||
|
|
@ -36,30 +36,30 @@ proc flushBuild(tokStream: var nlTokStream): nlTok =
|
||||||
# This indicates that the build token should inherit
|
# This indicates that the build token should inherit
|
||||||
# the nlTokType of the nlLStream's next character.
|
# the nlTokType of the nlLStream's next character.
|
||||||
proc isUntypedBuild(tokStream: nlTokStream): bool =
|
proc isUntypedBuild(tokStream: nlTokStream): bool =
|
||||||
result = isTokUntyped(tokStream.build.tokType)
|
result = isTokUntyped(tokStream.build.tType)
|
||||||
|
|
||||||
# Check whether an nlTokType is "compatible" with
|
# Check whether an nlTokType is "compatible" with
|
||||||
# the build token. flushBuild() should be called
|
# the build token. flushBuild() should be called
|
||||||
# when an incompatible token is discovered.
|
# when an incompatible token is discovered.
|
||||||
proc isCompatibleBuild(tokStream: nlTokStream, tokType: nlTokType): bool =
|
proc isCompatibleBuild(tokStream: nlTokStream, tType: nlTokType): bool =
|
||||||
result = (tokType == tokStream.build.tokType)
|
result = (tType == tokStream.build.tType)
|
||||||
|
|
||||||
# Add a character to the nlTokStream's build token.
|
# Add a character to the nlTokStream's build token.
|
||||||
# Returns a bool indicating if a new nlTok has been built
|
# Returns a bool indicating if a new nlTok has been built
|
||||||
# or not. flushBuild should then be called.
|
# or not. flushBuild should then be called.
|
||||||
proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] =
|
proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] =
|
||||||
let tokType = getTokType(c)
|
let tType = getTokType(c)
|
||||||
# check whether build token should inherit type
|
# check whether build token should inherit type
|
||||||
if isUntypedBuild(tokStream):
|
if isUntypedBuild(tokStream):
|
||||||
tokStream.build.tokType = tokType
|
tokStream.build.tType = tType
|
||||||
# check character and build token compatability
|
# check character and build token compatability
|
||||||
elif not isCompatibleBuild(tokStream, tokType):
|
elif not isCompatibleBuild(tokStream, tType):
|
||||||
# return flushed build token, and reset
|
# return flushed build token, and reset
|
||||||
result = some(flushBuild(tokStream))
|
result = some(flushBuild(tokStream))
|
||||||
# new build token is untyped so inherit type
|
# new build token is untyped so inherit type
|
||||||
tokStream.build.tokType = tokType
|
tokStream.build.tType = tType
|
||||||
# check if \0 terminator reached
|
# check if \0 terminator reached
|
||||||
elif isTokTerm(tokStream.build.tokType):
|
elif isTokTerm(tokStream.build.tType):
|
||||||
# return immediately to avoid concatinating '\0'
|
# return immediately to avoid concatinating '\0'
|
||||||
return some(flushBuild(tokStream))
|
return some(flushBuild(tokStream))
|
||||||
# else return none to indicate no build was completed
|
# else return none to indicate no build was completed
|
||||||
|
|
@ -69,7 +69,7 @@ proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] =
|
||||||
tokStream.build.lit.add(c)
|
tokStream.build.lit.add(c)
|
||||||
|
|
||||||
# Generates and returns the next token in the stream,
|
# Generates and returns the next token in the stream,
|
||||||
# result.tokType == nlTokType.NTERM implies line ended
|
# result.tType == nlTokType.NTERM implies line ended
|
||||||
proc nextTok(tokStream: var nlTokStream): nlTok =
|
proc nextTok(tokStream: var nlTokStream): nlTok =
|
||||||
# try progress to next char, receives none option on failure
|
# try progress to next char, receives none option on failure
|
||||||
for optchar in iterChars(tokStream.lstream):
|
for optchar in iterChars(tokStream.lstream):
|
||||||
|
|
@ -96,7 +96,7 @@ iterator toks*(tokStream: var nlTokStream): nlTok =
|
||||||
tok = nextTok(tokStream)
|
tok = nextTok(tokStream)
|
||||||
# \0 terminator means the line ended OR the file
|
# \0 terminator means the line ended OR the file
|
||||||
# has ended, so always yield a line-feed just in case
|
# has ended, so always yield a line-feed just in case
|
||||||
if isTokTerm(tok.tokType):
|
if isTokTerm(tok.tType):
|
||||||
yield tokTermToLineFeed(tok)
|
yield tokTermToLineFeed(tok)
|
||||||
break
|
break
|
||||||
yield tok
|
yield tok
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue