diff --git a/noether.nimble b/noether.nimble index e6c4a12..3f4ffdc 100644 --- a/noether.nimble +++ b/noether.nimble @@ -9,5 +9,6 @@ installExt = @["nim"] bin = @["noether", "nlx"] backend = "c" + # Dependencies requires "nim >= 2.2.0" diff --git a/src/nlx.nim b/src/nlx.nim index 1c6c446..b79cd2e 100644 --- a/src/nlx.nim +++ b/src/nlx.nim @@ -1,5 +1,5 @@ import os -import noether/lexer/tokstream +import noether/lex when isMainModule: echo "Noether Lang Extras v0.1.0 - nlx" diff --git a/src/noether/lexer/tokstream.nim b/src/noether/lex.nim similarity index 85% rename from src/noether/lexer/tokstream.nim rename to src/noether/lex.nim index 5ae2f65..443d3fb 100644 --- a/src/noether/lexer/tokstream.nim +++ b/src/noether/lex.nim @@ -1,5 +1,7 @@ include lstream +import os # TEMP import + type # Provides a stream-like interface for lexing nlToks # Internally reliant on the functionality of nlLStream @@ -8,7 +10,7 @@ type build: nlTok # the current token we're building # Resets the build token to an "empty" nlTok where -# only tType, lit, and startPos are initialised. +# only tokType, lit, and startPos are initialised. proc resetBuild(tokStream: var nlTokStream) = tokStream.build = emptyTok(tokStream.lstream.pos) @@ -18,7 +20,7 @@ proc resetBuild(tokStream: var nlTokStream) = proc finishBuild(tokStream: var nlTokStream) = # if we've reached \0 terminator then forge the start # and end positions to point OUTSIDE the line - let endPos = if isTokTerm(tokStream.build.tType): + let endPos = if isTokTerm(tokStream.build.tokType): inc tokStream.build.startPos; tokStream.build.startPos else: Natural tokStream.lstream.pos @@ -36,30 +38,30 @@ proc flushBuild(tokStream: var nlTokStream): nlTok = # This indicates that the build token should inherit # the nlTokType of the nlLStream's next character. proc isUntypedBuild(tokStream: nlTokStream): bool = - result = isTokUntyped(tokStream.build.tType) + result = isTokUntyped(tokStream.build.tokType) # Check whether an nlTokType is "compatible" with # the build token. flushBuild() should be called # when an incompatible token is discovered. -proc isCompatibleBuild(tokStream: nlTokStream, tType: nlTokType): bool = - result = (tType == tokStream.build.tType) +proc isCompatibleBuild(tokStream: nlTokStream, tokType: nlTokType): bool = + result = (tokType == tokStream.build.tokType) # Add a character to the nlTokStream's build token. # Returns a bool indicating if a new nlTok has been built # or not. flushBuild should then be called. proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] = - let tType = getTokType(c) + let tokType = getTokType(c) # check whether build token should inherit type if isUntypedBuild(tokStream): - tokStream.build.tType = tType + tokStream.build.tokType = tokType # check character and build token compatability - elif not isCompatibleBuild(tokStream, tType): + elif not isCompatibleBuild(tokStream, tokType): # return flushed build token, and reset result = some(flushBuild(tokStream)) # new build token is untyped so inherit type - tokStream.build.tType = tType + tokStream.build.tokType = tokType # check if \0 terminator reached - elif isTokTerm(tokStream.build.tType): + elif isTokTerm(tokStream.build.tokType): # return immediately to avoid concatinating '\0' return some(flushBuild(tokStream)) # else return none to indicate no build was completed @@ -69,7 +71,7 @@ proc appendBuild(tokStream: var nlTokStream, c: char): Option[nlTok] = tokStream.build.lit.add(c) # Generates and returns the next token in the stream, -# result.tType == nlTokType.NTERM implies line ended +# result.tokType == nlTokType.NTERM implies line ended proc nextTok(tokStream: var nlTokStream): nlTok = # try progress to next char, receives none option on failure for optchar in iterChars(tokStream.lstream): @@ -96,7 +98,7 @@ iterator toks*(tokStream: var nlTokStream): nlTok = tok = nextTok(tokStream) # \0 terminator means the line ended OR the file # has ended, so always yield a line-feed just in case - if isTokTerm(tok.tType): + if isTokTerm(tok.tokType): yield tokTermToLineFeed(tok) break yield tok diff --git a/src/noether/lexer/lstream.nim b/src/noether/lstream.nim similarity index 98% rename from src/noether/lexer/lstream.nim rename to src/noether/lstream.nim index 862eb7b..5bc79f2 100644 --- a/src/noether/lexer/lstream.nim +++ b/src/noether/lstream.nim @@ -1,7 +1,7 @@ import std/streams import std/options -include tok +include tokens type # Character streaming for the nlTokStream diff --git a/src/noether/parser/arborist.nim b/src/noether/parser/arborist.nim deleted file mode 100644 index 42888c8..0000000 --- a/src/noether/parser/arborist.nim +++ /dev/null @@ -1,7 +0,0 @@ -# Attempt to form an nlAST from a nlTokStream -proc arborise(tokStream: nlTokStream): nlNode = - for tok in toks(tokStream): - case tok.tokType: - of nlTokType.DQUO: - # Attempt to parse string literal - parse_strl() diff --git a/src/noether/parser/nodes.nim b/src/noether/parser/nodes.nim deleted file mode 100644 index fdb78e1..0000000 --- a/src/noether/parser/nodes.nim +++ /dev/null @@ -1,18 +0,0 @@ -from ../lexer/tok import nlTok -from ../lexer/tokstraem import - -type - # NOTE: by the end of parsing NO nodes should - # NOTE: have nlNodeType.NONE - nlNodeType = enum - NONE, # Placeholder Value - TERM, # Indicates the tree has terminated - STRL, # String Literal - CHRL, # Character Literal - nlNode {.acyclic.} = ref object of RootObj - nType: nlNodeType - toks: seq[nlTok] # nodes store the tokens that build them - left, right: nlNode - -proc parse() - diff --git a/src/noether/lexer/tok.nim b/src/noether/tokens.nim similarity index 86% rename from src/noether/lexer/tok.nim rename to src/noether/tokens.nim index 3b2464b..6e18151 100644 --- a/src/noether/lexer/tok.nim +++ b/src/noether/tokens.nim @@ -23,10 +23,9 @@ type SQUO, # ' Single Quotation Marking DQUO, # " Double Quotation Marking GRVA, # ` Grave Accent - HASH, # # Number Sign (Hashtag) nlTok = object - tType*: nlTokType + tokType*: nlTokType lit*: string line*: Natural startPos*: Natural @@ -34,23 +33,23 @@ type # Generates an "empty" nlTok with only a startPos, # all other fields are expected to be filled out later. -# NOTE: tType initialised to nlTokType.NUL +# NOTE: tokType initialised to nlTokType.NUL # NOTE: lit initialised to empty string # NOTE: all other fields are uninitialised proc emptyTok(startPos: int): nlTok = result = nlTok( - tType: nlTokType.NONE, + tokType: nlTokType.NONE, lit: "", startPos: Natural startPos, ) # Checks if an nlTok has nlTokType.NONE -proc isTokUntyped(tType: nlTokType): bool = - result = (tType == nlTokType.NONE) +proc isTokUntyped(tokType: nlTokType): bool = + result = (tokType == nlTokType.NONE) # Checks if an nlTok has nlTokType.TERM -proc isTokTerm(tType: nlTokType): bool = - result = (tType == nlTokType.TERM) +proc isTokTerm(tokType: nlTokType): bool = + result = (tokType == nlTokType.TERM) # This method is only used to convert null # terminator nlToks into line-feed ones. @@ -59,7 +58,7 @@ proc isTokTerm(tType: nlTokType): bool = # NOTE: strings in a useful but annoying way proc tokTermToLineFeed(tok: nlTok): nlTok = result = nlTok( - tType: nlTokType.LNFD, + tokType: nlTokType.LNFD, lit: tok.lit, line: tok.line, startPos: tok.startPos, @@ -93,7 +92,5 @@ proc getTokType(c: char): nlTokType = result = nlTokType.DQUO of '`': result = nlTokType.GRVA - of '#': - result = nlTokType.HASH else: result = nlTokType.WORD