metrale_grammar/regex/
mod.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//
3// Regex converter — port of `cpp/regex_converter.{h,cc}`.
4//
5// Converts a JavaScript-flavoured regex pattern string into an EBNF
6// grammar string (the `RegexToEBNF` entry point) and, as a
7// convenience, into a parsed `GrammarData` AST by feeding that EBNF
8// through the existing `grammar::parse_ebnf` parser.
9//
10// RELATION TO THE FSM REGEX BUILDER
11// ---------------------------------
12// `fsm::builder::RegexFsmBuilder` (ported by the W2 FSM agent) takes a
13// regex and builds an NFA/FSM directly. This module is the *other*
14// path: it lowers a regex to the EBNF grammar surface syntax, which
15// the EBNF parser then turns into a `GrammarData` AST. The two share
16// no code — they are independent lowerings of the same input, exactly
17// as in upstream xgrammar where `regex_converter.cc` and
18// `fsm_builder.cc` are separate translation units.
19//
20// The implementation refers to the regex described in
21// <https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions>.
22//
23// FAITHFULNESS / SIMPLIFICATIONS vs C++
24// -------------------------------------
25//  * C++ aborts the process (`XGRAMMAR_LOG(FATAL)`) on a malformed
26//    regex; we return `Err(RegexError)` instead — no panics, no
27//    `unsafe`.
28//  * C++ emits warnings to stderr for stray `^`/`$` and unknown
29//    escapes; we collect them into `ConvertOutput::warnings` rather
30//    than performing I/O (keeps the converter pure).
31//  * Backreferences, lookahead/lookbehind, named-group *flags*,
32//    unicode-property `\p{…}` and word boundaries are unsupported and
33//    reported as errors — identical to upstream.
34
35mod converter;
36mod cursor;
37mod escape_handlers;
38mod sub_handlers;
39
40pub use converter::{ConvertOutput, RegexConverter};
41
42use crate::grammar::{GrammarData, parse_ebnf};
43
44/// Error raised while converting a regex to EBNF.
45///
46/// `position` is 1-based, matching the C++ `RaiseError` message
47/// (`current_ - start_ + 1`).
48#[derive(Debug, Clone, PartialEq, Eq)]
49pub struct RegexError {
50    /// 1-based codepoint offset into the regex where the error was
51    /// detected.
52    pub position: usize,
53    /// Human-readable description of the problem.
54    pub message: String,
55}
56
57impl std::fmt::Display for RegexError {
58    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
59        write!(
60            f,
61            "Regex parsing error at position {}: {}",
62            self.position, self.message
63        )
64    }
65}
66
67impl std::error::Error for RegexError {}
68
69/// Convert a regex string to an EBNF grammar string.
70///
71/// When `with_rule_name` is `true` the result is wrapped as
72/// `root ::= <body>\n`; otherwise just the rule body is returned.
73/// Faithful port of C++ `RegexToEBNF`.
74pub fn regex_to_ebnf(regex: &str, with_rule_name: bool) -> Result<String, RegexError> {
75    let body = RegexConverter::new(regex)?.convert()?.ebnf;
76    if with_rule_name {
77        Ok(format!("root ::= {body}\n"))
78    } else {
79        Ok(body)
80    }
81}
82
83/// Error from [`regex_to_grammar`]: either the regex itself was
84/// malformed, or the EBNF it lowered to failed to parse.
85#[derive(Debug)]
86pub enum RegexToGrammarError {
87    /// The regex pattern was malformed.
88    Regex(RegexError),
89    /// The lowered EBNF failed to parse (should not happen for a
90    /// well-formed regex; indicates an internal converter bug).
91    Ebnf(crate::grammar::ParseError),
92}
93
94impl std::fmt::Display for RegexToGrammarError {
95    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
96        match self {
97            RegexToGrammarError::Regex(e) => write!(f, "{e}"),
98            RegexToGrammarError::Ebnf(e) => write!(f, "lowered EBNF failed to parse: {e:?}"),
99        }
100    }
101}
102
103impl std::error::Error for RegexToGrammarError {}
104
105/// Convert a regex string straight to a parsed [`GrammarData`] AST.
106///
107/// This is the regex→grammar-AST path: it lowers the regex to EBNF
108/// (via [`regex_to_ebnf`]) and parses that with the existing EBNF
109/// parser, so the result is a fully-formed grammar usable by the rest
110/// of the crate. Upstream xgrammar's `Grammar::FromRegex` does the
111/// same composition.
112pub fn regex_to_grammar(regex: &str) -> Result<GrammarData, RegexToGrammarError> {
113    let ebnf = regex_to_ebnf(regex, true).map_err(RegexToGrammarError::Regex)?;
114    parse_ebnf(&ebnf, "root").map_err(RegexToGrammarError::Ebnf)
115}
116
117#[cfg(test)]
118mod tests;
119#[cfg(test)]
120mod tests_patterns;
metrale_grammar/regex/mod.rs

metrale_grammar/regex/
mod.rs
