Add dump_tokens.py

dump_tokens.py

@@ -0,0 +1,10 @@
+import sys
+
+from ppp_lexer import Lexer
+from ppp_tokens import EofToken
+
+_program, file_path = sys.argv
+
+lexer = Lexer.from_file(file_path)
+
+while not lexer.check_token(EofToken()): print(lexer.next_token())

ppp_lexer.py

@@ -60,9 +60,9 @@ class Lexer:
 						return self._token(loc, word, IdentifierToken(word))
 			case '"':
 				# TODO: Proper escaping
+				loc = self._loc()
 				self._advance()
 				start_location = self._location
-				loc = self._loc()
 				escaping = False
 				while self._location < len(self._source) and (self._source[self._location] != '"' or escaping):
 					escaping = self._source[self._location] == '\\' if not escaping else False

ppp_tokens.py

@@ -106,3 +106,6 @@ class Token:
 	loc: Location
 	value: str
 	contents: TokenContents
+
+	def __repr__(self) -> str:
+		return f"{self.loc}: {self.contents}"