Fix token location reporting
This commit is contained in:
parent
b02ca87760
commit
18b22cd5d1
SSH Key Fingerprint:
SHA256:N3w+8798IMWBt7SYH8G1C0iJlIa2HIIcRCXwILT5FvM
106
ppp_lexer.py
106
ppp_lexer.py
@ -1,31 +1,32 @@
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from ppp_tokens import EofToken, IdentifierToken, Keyword, KeywordToken, NumberToken, StringToken, Symbol, SymbolToken, Token, TokenContents
|
||||
from ppp_tokens import EofToken, IdentifierToken, Keyword, KeywordToken, NumberToken, StringToken, Symbol, SymbolToken, Token, TokenContents, Location
|
||||
|
||||
class Lexer:
|
||||
def __init__(self, source: str) -> None:
|
||||
def __init__(self, source: str, filename: str) -> None:
|
||||
self._source = source
|
||||
self._location = 0
|
||||
self._line = 1
|
||||
self._col = 0
|
||||
self._filename = filename
|
||||
self._peeked_token: Optional[Token] = None
|
||||
self._current: str = ""
|
||||
|
||||
def _loc(self) -> Location:
|
||||
return Location(self._filename, self._line, self._col)
|
||||
|
||||
def _token(self, loc: Location, value: str, contents: TokenContents) -> Token:
|
||||
return Token(loc, value, contents)
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: str) -> 'Lexer':
|
||||
with open(path) as f:
|
||||
return cls(f.read())
|
||||
return cls(f.read(), path)
|
||||
|
||||
def _advance(self) -> str:
|
||||
def _advance(self):
|
||||
assert self._location < len(self._source)
|
||||
self._line, self._col = (self._line + 1, 0) if self._current == '\n' else (self._line, self._col + 1)
|
||||
self._line, self._col = (self._line + 1, 0) if self._source[self._location] == '\n' else (self._line, self._col + 1)
|
||||
self._location += 1
|
||||
self._current = self._source[self._location] if self._location < len(self._source) else ''
|
||||
return self._current
|
||||
|
||||
# def _peek(self) -> str:
|
||||
# assert self._location < len(self._source)-1
|
||||
|
||||
def next_token(self) -> Token:
|
||||
if self._peeked_token is not None:
|
||||
@ -34,71 +35,84 @@ class Lexer:
|
||||
|
||||
while self._location < len(self._source) and self._source[self._location] in ' \t\n': self._advance()
|
||||
|
||||
if self._location >= len(self._source): return Token(self._line, self._col, '\0', EofToken())
|
||||
if self._location >= len(self._source): return self._token(self._loc(), '\0', EofToken())
|
||||
|
||||
match self._source[self._location]:
|
||||
case c if c.isdigit():
|
||||
start_location = self._location
|
||||
while self._location < len(self._source) and self._source[self._location].isdigit(): self._location += 1
|
||||
loc = self._loc()
|
||||
while self._location < len(self._source) and self._source[self._location].isdigit(): self._advance()
|
||||
number = int(self._source[start_location:self._location])
|
||||
return Token(self._line, self._col, self._source[start_location:self._location], NumberToken(number))
|
||||
return self._token(loc, self._source[start_location:self._location], NumberToken(number))
|
||||
case c if c.isalpha() or c == "_":
|
||||
start_location = self._location
|
||||
while self._location < len(self._source) and (self._source[self._location].isalpha() or self._source[self._location] in '_'): self._location += 1
|
||||
loc = self._loc()
|
||||
while self._location < len(self._source) and (self._source[self._location].isalpha() or self._source[self._location] in '_'): self._advance()
|
||||
word = self._source[start_location:self._location]
|
||||
try:
|
||||
keyword = Keyword(word)
|
||||
return Token(self._line, self._col, word, KeywordToken(keyword))
|
||||
return self._token(loc, word, KeywordToken(keyword))
|
||||
except ValueError:
|
||||
try:
|
||||
symbol = Symbol(word)
|
||||
return Token(self._line, self._col, word, SymbolToken(symbol))
|
||||
return self._token(loc, word, SymbolToken(symbol))
|
||||
except ValueError:
|
||||
return Token(self._line, self._col, word, IdentifierToken(word))
|
||||
return self._token(loc, word, IdentifierToken(word))
|
||||
case '"':
|
||||
# TODO: Escaping
|
||||
self._location += 1
|
||||
# TODO: Proper escaping
|
||||
self._advance()
|
||||
start_location = self._location
|
||||
loc = self._loc()
|
||||
escaping = False
|
||||
while self._location < len(self._source) and (self._source[self._location] != '"' or escaping):
|
||||
escaping = self._source[self._location] == '\\' if not escaping else False
|
||||
self._location += 1
|
||||
self._advance()
|
||||
string = self._source[start_location:self._location].encode('utf-8').decode('unicode_escape')
|
||||
self._location += 1
|
||||
return Token(self._line, self._col, self._source[start_location-1:self._location], StringToken(string))
|
||||
self._advance()
|
||||
return self._token(loc, self._source[start_location-1:self._location], StringToken(string))
|
||||
# TODO: Make a proper Trie for this.
|
||||
case '|' if self._location < len(self._source)-1 and self._source[self._location+1] == '|':
|
||||
self._location += 2
|
||||
return Token(self._line, self._col, self._source[self._location-2:self._location], SymbolToken(Symbol.Dpipe))
|
||||
loc = self._loc()
|
||||
self._advance(); self._advance()
|
||||
return self._token(loc, self._source[self._location-2:self._location], SymbolToken(Symbol.Dpipe))
|
||||
case '&' if self._location < len(self._source)-1 and self._source[self._location+1] == '&':
|
||||
self._location += 2
|
||||
return Token(self._line, self._col, self._source[self._location-2:self._location], SymbolToken(Symbol.Dampersand))
|
||||
loc = self._loc()
|
||||
self._advance(); self._advance()
|
||||
return self._token(loc, self._source[self._location-2:self._location], SymbolToken(Symbol.Dampersand))
|
||||
case '*' if self._location < len(self._source)-1 and self._source[self._location+1] == '*':
|
||||
self._location += 2
|
||||
return Token(self._line, self._col, self._source[self._location-2:self._location], SymbolToken(Symbol.Dasterisk))
|
||||
loc = self._loc()
|
||||
self._advance(); self._advance()
|
||||
return self._token(loc, self._source[self._location-2:self._location], SymbolToken(Symbol.Dasterisk))
|
||||
case '-' if self._location < len(self._source)-1 and self._source[self._location+1] == '>':
|
||||
self._location += 2
|
||||
return Token(self._line, self._col, self._source[self._location-2:self._location], SymbolToken(Symbol.Arrow))
|
||||
loc = self._loc()
|
||||
self._advance(); self._advance()
|
||||
return self._token(loc, self._source[self._location-2:self._location], SymbolToken(Symbol.Arrow))
|
||||
case '>' if self._location < len(self._source)-1 and self._source[self._location+1] == '=':
|
||||
self._location += 2
|
||||
return Token(self._line, self._col, self._source[self._location-2:self._location], SymbolToken(Symbol.GreaterEqual))
|
||||
loc = self._loc()
|
||||
self._advance(); self._advance()
|
||||
return self._token(loc, self._source[self._location-2:self._location], SymbolToken(Symbol.GreaterEqual))
|
||||
case '<' if self._location < len(self._source)-1 and self._source[self._location+1] == '=':
|
||||
self._location += 2
|
||||
return Token(self._line, self._col, self._source[self._location-2:self._location], SymbolToken(Symbol.LesserEqual))
|
||||
loc = self._loc()
|
||||
self._advance(); self._advance()
|
||||
return self._token(loc, self._source[self._location-2:self._location], SymbolToken(Symbol.LesserEqual))
|
||||
case '=' if self._location < len(self._source)-1 and self._source[self._location+1] == '=':
|
||||
self._location += 2
|
||||
return Token(self._line, self._col, self._source[self._location-2:self._location], SymbolToken(Symbol.Dequal))
|
||||
loc = self._loc()
|
||||
self._advance(); self._advance()
|
||||
return self._token(loc, self._source[self._location-2:self._location], SymbolToken(Symbol.Dequal))
|
||||
case '=' if self._location < len(self._source)-1 and self._source[self._location+1] == '>':
|
||||
self._location += 2
|
||||
return Token(self._line, self._col, self._source[self._location-2:self._location], SymbolToken(Symbol.EqualArrow))
|
||||
loc = self._loc()
|
||||
self._advance(); self._advance()
|
||||
return self._token(loc, self._source[self._location-2:self._location], SymbolToken(Symbol.EqualArrow))
|
||||
case '!' if self._location < len(self._source)-1 and self._source[self._location+1] == '=':
|
||||
self._location += 2
|
||||
return Token(self._line, self._col, self._source[self._location-2:self._location], SymbolToken(Symbol.NotEqual))
|
||||
loc = self._loc()
|
||||
self._advance(); self._advance()
|
||||
return self._token(loc, self._source[self._location-2:self._location], SymbolToken(Symbol.NotEqual))
|
||||
case c if c in Symbol._value2member_map_:
|
||||
self._location += 1
|
||||
return Token(self._line, self._col, self._source[self._location-1], SymbolToken(Symbol(c)))
|
||||
loc = self._loc()
|
||||
self._advance()
|
||||
return self._token(loc, self._source[self._location-1], SymbolToken(Symbol(c)))
|
||||
case _:
|
||||
assert False, ("Unimplemented", c, self._location)
|
||||
raise SyntaxError(f"{self._loc()}: Unknown character: '{c}'")
|
||||
assert False, "Unreachable"
|
||||
|
||||
def peek_token(self) -> Token:
|
||||
@ -108,12 +122,12 @@ class Lexer:
|
||||
|
||||
def assert_tokenkind(self, kind: type) -> Token:
|
||||
token = self.next_token()
|
||||
assert isinstance(token.contents, kind), (f"Expected {kind} but got {token.contents}!", self.next_token(), self.next_token(), self.next_token())
|
||||
if not isinstance(token.contents, kind): raise SyntaxError(f"{token.loc}: Expected {kind} but got {token.contents}!")
|
||||
return token
|
||||
|
||||
def assert_token(self, expected: TokenContents) -> Token:
|
||||
token = self.next_token()
|
||||
assert token.contents == expected, (f"Expected {expected} but got {token.contents}!", self.next_token(), self.next_token())
|
||||
if token.contents != expected: raise SyntaxError(f"{token.loc}: Expected {expected} but got {token.contents}!")
|
||||
return token
|
||||
|
||||
def check_token(self, expected: TokenContents) -> bool:
|
||||
|
||||
@ -89,8 +89,16 @@ class EofToken: pass
|
||||
TokenContents = KeywordToken | IdentifierToken | NumberToken | StringToken | SymbolToken | EofToken
|
||||
|
||||
@dataclass
|
||||
class Token:
|
||||
class Location:
|
||||
file: str
|
||||
line: int
|
||||
col: int
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.file}:{self.line}:{self.col+1}"
|
||||
|
||||
@dataclass
|
||||
class Token:
|
||||
loc: Location
|
||||
value: str
|
||||
contents: TokenContents
|
||||
|
||||
Loading…
Reference in New Issue
Block a user