python-plus-plus/ppp_tokens.py

from dataclasses import dataclass
from enum import Enum

class Keyword(Enum):
	Enum = 'enum'
	Struct = 'struct'
	Func = 'func'
	If = 'if'
	Else = 'else'
	While = 'while'
	Break = 'break'
	Continue = 'continue'
	Do = 'do'
	For = 'for'
	To = 'to'
	In = 'in'
	Match = 'match'
	Case = 'case'
	Assert = 'assert'
	Return = 'return'
	Lambda = 'lambda'
	Import = 'import'
	Type = 'type'
	Defer = 'defer'

	def __str__(self) -> str: return self._value_

class Symbol(Enum):
	Open = '('
	Close = ')'
	OpenCurly = '{'
	CloseCurly = '}'
	Comma = ','
	OpenSquare = '['
	CloseSquare = ']'
	Colon = ':'
	Left = '<'
	Right = '>'
	Arrow = '->'
	EqualArrow = '=>'
	Semicolon = ';'
	Equal = '='
	Dequal = '=='
	Exclamation = '!'
	NotEqual = '!='
	Dot = '.'
	Plus = '+'
	Dash = '-'
	Asterisk = '*'
	Dasterisk = '**'
	Slash = '/'
	QuestionMark = '?'
	Ampersand = '&'
	Dampersand = '&&'
	Pipe = '|'
	Dpipe = '||'
	Dleft = '<<'
	Dright = '>>'
	GreaterEqual = '>='
	LesserEqual = '<='
	Percent = '%'
	Tilde = '~'
	Carot = '^'

	def __str__(self) -> str: return self._value_

@dataclass
class KeywordToken:
	keyword: Keyword

@dataclass
class IdentifierToken:
	identifier: str

@dataclass
class NumberToken:
	number: int

@dataclass
class StringToken:
	string: str

@dataclass
class SymbolToken:
	symbol: Symbol

	def __hash__(self) -> int:
		return hash(('symbol', self.symbol))

@dataclass
class EofToken: pass

TokenContents = KeywordToken | IdentifierToken | NumberToken | StringToken | SymbolToken | EofToken

@dataclass
class Location:
	file: str
	line: int
	col: int

	def __repr__(self) -> str:
		return f"{self.file}:{self.line}:{self.col+1}"

@dataclass
class Token:
	loc: Location
	value: str
	contents: TokenContents

	def __repr__(self) -> str:
		return f"{self.loc}: {self.contents}"