import "ppp_tokens.ppp"; struct Lexer { source: str, location: int, line: int, col: int, peeked_token: OptionalToken } func new_lexer(source: str) -> Lexer { return Lexer{ source = source, location = 0, line = 1, col = 0, peeked_token = OptionalToken.None }; } func lexer_from_file(path: str) -> Lexer return new_lexer(read(path)); func is_space(char: str) -> bool { return char == " " || char == "\t" || char == "\n"; } func is_digit(char: str) -> bool { return char == "0" || char == "1" || char == "2" || char == "3" || char == "4" || char == "5" || char == "6" || char == "7" || char == "8" || char == "9"; } func is_alpha(char: str) -> bool { return char == "a" || char == "b" || char == "c" || char == "d" || char == "e" || char == "f" || char == "g" || char == "h" || char == "i" || char == "j" || char == "k" || char == "l" || char == "m" || char == "n" || char == "o" || char == "p" || char == "q" || char == "r" || char == "s" || char == "t" || char == "u" || char == "v" || char == "w" || char == "x" || char == "y" || char == "z" || char == "A" || char == "B" || char == "C" || char == "D" || char == "E" || char == "F" || char == "G" || char == "H" || char == "I" || char == "J" || char == "K" || char == "L" || char == "M" || char == "N" || char == "O" || char == "P" || char == "Q" || char == "R" || char == "S" || char == "T" || char == "U" || char == "V" || char == "W" || char == "X" || char == "Y" || char == "Z" || char == "_"; } func lexer_next_token(lexer: Lexer) -> Token { match lexer.peeked_token in { case Some(token) do { lexer.peeked_token = OptionalToken.None; return token; } } while lexer.location < len(lexer.source) && is_space(lexer.source[lexer.location]) do { if lexer.source[lexer.location] == "\n" do { lexer.line = lexer.line + 1; lexer.col = 0; } lexer.location = lexer.location + 1; } if lexer.location >= len(lexer.source) return Token{line=lexer.line, col=lexer.col, value="\0", contents=TokenContents.Eof}; if is_digit(lexer.source[lexer.location]) do { number_str: str = ""; while lexer.location < len(lexer.source) && is_digit(lexer.source[lexer.location]) do { number_str = number_str + lexer.source[lexer.location]; lexer.location = lexer.location + 1; } number: int = str_to_int(number_str); return Token{line=lexer.line, col=lexer.col, value=number_str, contents=TokenContents.Number(number)}; } else if is_alpha(lexer.source[lexer.location]) do { word_str: str = ""; while lexer.location < len(lexer.source) && is_alpha(lexer.source[lexer.location]) do { word_str = word_str + lexer.source[lexer.location]; lexer.location = lexer.location + 1; } match keyword_from_str(word_str) in { case Some(keyword) return Token{line=lexer.line, col=lexer.col, value=word_str, contents=TokenContents.Keyword(keyword)}; case None return Token{line=lexer.line, col=lexer.col, value=word_str, contents=TokenContents.Identifier(word_str)}; } assert false, "Identifier"; } else if lexer.source[lexer.location] == "\"" do { lexer.location = lexer.location + 1; string_str: str = ""; escaping: bool = false; while lexer.location < len(lexer.source) && (lexer.source[lexer.location] != "\"" || escaping) do { escaping = escaping? false: lexer.source[lexer.location] == "\\"; string_str = string_str + lexer.source[lexer.location]; lexer.location = lexer.location + 1; } lexer.location = lexer.location + 1; return Token{line=lexer.line, col=lexer.col, value="\""+string_str+"\"", contents=TokenContents.String(string_str)}; } else if lexer.source[lexer.location] == "|" && lexer.location < len(lexer.source)-1 && lexer.source[lexer.location+1] == "|" do { lexer.location = lexer.location + 2; return Token{line=lexer.line, col=lexer.col, value="||", contents=TokenContents.Symbol(Symbol.Dpipe)}; } else if lexer.source[lexer.location] == "&" && lexer.location < len(lexer.source)-1 && lexer.source[lexer.location+1] == "&" do { lexer.location = lexer.location + 2; return Token{line=lexer.line, col=lexer.col, value="&&", contents=TokenContents.Symbol(Symbol.Dampersand)}; } else if lexer.source[lexer.location] == "*" && lexer.location < len(lexer.source)-1 && lexer.source[lexer.location+1] == "*" do { lexer.location = lexer.location + 2; return Token{line=lexer.line, col=lexer.col, value="**", contents=TokenContents.Symbol(Symbol.Dasterisk)}; } else if lexer.source[lexer.location] == "-" && lexer.location < len(lexer.source)-1 && lexer.source[lexer.location+1] == ">" do { lexer.location = lexer.location + 2; return Token{line=lexer.line, col=lexer.col, value="->", contents=TokenContents.Symbol(Symbol.Arrow)}; } else if lexer.source[lexer.location] == ">" && lexer.location < len(lexer.source)-1 && lexer.source[lexer.location+1] == "=" do { lexer.location = lexer.location + 2; return Token{line=lexer.line, col=lexer.col, value=">=", contents=TokenContents.Symbol(Symbol.GreaterEqual)}; } else if lexer.source[lexer.location] == "<" && lexer.location < len(lexer.source)-1 && lexer.source[lexer.location+1] == "=" do { lexer.location = lexer.location + 2; return Token{line=lexer.line, col=lexer.col, value="<=", contents=TokenContents.Symbol(Symbol.LesserEqual)}; } else if lexer.source[lexer.location] == "=" && lexer.location < len(lexer.source)-1 && lexer.source[lexer.location+1] == "=" do { lexer.location = lexer.location + 2; return Token{line=lexer.line, col=lexer.col, value="==", contents=TokenContents.Symbol(Symbol.Dequal)}; } else if lexer.source[lexer.location] == "!" && lexer.location < len(lexer.source)-1 && lexer.source[lexer.location+1] == "=" do { lexer.location = lexer.location + 2; return Token{line=lexer.line, col=lexer.col, value="!=", contents=TokenContents.Symbol(Symbol.NotEqual)}; } else { match symbol_from_str(lexer.source[lexer.location]) in { case Some(symbol) do { lexer.location = lexer.location + 1; return Token{line=lexer.line, col=lexer.col, value=lexer.source[lexer.location-1], contents=TokenContents.Symbol(symbol)}; } case None assert False, "Unimplemented, '%s'" % lexer.source[lexer.location]; } } } func lexer_peek_token(lexer: Lexer) -> Token { match lexer.peeked_token in { case Some(token) return token; case None do { token: Token = lexer_next_token(lexer); lexer.peeked_token = OptionalToken.Some(token); return token; } } } func lexer_check_token(lexer: Lexer, expected: TokenContents) -> bool { token: Token = lexer_peek_token(lexer); return token.contents == expected; } func lexer_take_token(lexer: Lexer, token: TokenContents) -> OptionalToken { if lexer_check_token(lexer, token) return OptionalToken.Some(lexer_next_token(lexer)); return OptionalToken.None; } func lexer_take_tokens(lexer: Lexer, tokens: TokenContents[]) -> OptionalToken { for token in tokens do { if lexer_check_token(lexer, token) return OptionalToken.Some(lexer_next_token(lexer)); } return OptionalToken.None; } func lexer_assert_token(lexer: Lexer, expected: TokenContents) -> Token { token: Token = lexer_next_token(lexer); assert token.contents == expected, "Expected %s but got %s!" % (token_contents_to_str(expected), token_to_str(token)); return token; } func lexer_check_tokens(lexer: Lexer, tokens: TokenContents[]) -> bool { for token in tokens if lexer_check_token(lexer, token) return true; return false; }