metrale_grammar/grammar/
lexer_token.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//
3// EBNF token type definitions. Split out of `lexer.rs` to keep each
4// file under the 250-line cap. Port of `EBNFLexer::TokenType` and
5// `EBNFLexer::Token` from xgrammar `cpp/grammar_parser.h`.
6
7/// EBNF token kinds. Port of `EBNFLexer::TokenType`.
8#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9pub enum TokenType {
10    /// Name on the left of `::=`, e.g. `root`.
11    RuleName,
12    /// A reference to a rule, or a macro name (e.g. `TagDispatch`).
13    Identifier,
14    /// A `"..."` string literal.
15    StringLiteral,
16    /// `true` / `false`.
17    BooleanLiteral,
18    /// An integer literal.
19    IntegerLiteral,
20    /// `(`
21    LParen,
22    /// `)`
23    RParen,
24    /// `{`
25    LBrace,
26    /// `}`
27    RBrace,
28    /// `|`
29    Pipe,
30    /// `,`
31    Comma,
32    /// End of input.
33    EndOfFile,
34    /// `::=`
35    Assign,
36    /// `=`
37    Equal,
38    /// `*`
39    Star,
40    /// `+`
41    Plus,
42    /// `?`
43    Question,
44    /// `[`
45    LBracket,
46    /// `]`
47    RBracket,
48    /// `-` inside a character class.
49    Dash,
50    /// `^` inside a character class.
51    Caret,
52    /// A single (possibly escaped) character inside a character class.
53    CharInCharClass,
54    /// A regex-style escape with special meaning inside a class (`\d`…).
55    EscapeInCharClass,
56    /// `(=` — opens a lookahead assertion.
57    LookaheadLParen,
58}
59
60/// The processed value carried by a [`Token`].
61#[derive(Debug, Clone, PartialEq)]
62pub enum TokenValue {
63    /// No processed value.
64    None,
65    /// String value (string literals; identifier names; class escapes).
66    Str(String),
67    /// Integer value (integer literals).
68    Int(i64),
69    /// Boolean value (`true`/`false`).
70    Bool(bool),
71    /// Codepoint value (a character inside a character class).
72    Codepoint(i32),
73}
74
75/// A single lexer token. Port of `EBNFLexer::Token`.
76#[derive(Debug, Clone, PartialEq)]
77pub struct Token {
78    /// The token kind.
79    pub ty: TokenType,
80    /// Original source text of the token.
81    pub lexeme: String,
82    /// Processed value.
83    pub value: TokenValue,
84    /// 1-based source line.
85    pub line: i32,
86    /// 1-based source column.
87    pub column: i32,
88}
89
90/// A lexer error with source position.
91#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
92#[error("EBNF lexer error at line {line}, column {column}: {msg}")]
93pub struct LexError {
94    /// 1-based source line.
95    pub line: i32,
96    /// 1-based source column.
97    pub column: i32,
98    /// Human-readable error message.
99    pub msg: String,
100}
metrale_grammar/grammar/lexer_token.rs

metrale_grammar/grammar/
lexer_token.rs
