package main TokenKind :: enum { Invalid, EOF, // Literals Identifier, Integer, Float, String, Character, // Keywords Function, Struct, Enum, Union, Type, Use, Pub, Mut, As, BitwiseAs, In, If, Else, Elif, For, Break, Continue, Switch, Case, Ret, Static, Defer, Let, // Logical Operators Keywords And, Or, // Operators Add, Subtract, Multiply, Divide, Modulo, Exponent, Assign, Increment, Decrement, // Logical Operators Equals, NotEquals, LessThan, GreaterThan, LessThanOrEqual, GreaterThanOrEqual, Not, // Bitwise Operators BitwiseAnd, BitwiseOr, BitwiseXOR, BitwiseNot, BitwiseLeftShift, BitwiseRightShift, // Delimiters OpenParen, CloseParen, OpenBrace, CloseBrace, OpenBracket, CloseBracket, // Punctuation Question, // For Zig-like error handling Colon, Arrow, Dot, Comma, // Other Semicolon, } TextPosition :: struct { line: u64, column: u64, } TextRange :: struct { start: TextPosition, end: TextPosition, } TokenValue :: union { bool, u64, f64, [dynamic]u8, ^u8, } Token :: struct { kind: TokenKind, value: TokenValue, range: TextRange, } token_create :: proc(kind: TokenKind, range: TextRange) -> Token { return { kind = kind, value = nil, range = range, }; } token_create_u8 :: proc(kind: TokenKind, text: [dynamic]u8, range: TextRange) -> Token { return { kind = kind, value = text, range = range, }; } token_create_u64 :: proc(kind: TokenKind, value: u64, range: TextRange) -> Token { return { kind = kind, value = value, range = range, }; } token_create_f64 :: proc(kind: TokenKind, value: f64, range: TextRange) -> Token { return { kind = kind, value = value, range = range, }; }