metrale_grammar/
lib.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//
3// Pure-Rust XGrammar — a from-scratch port of mlc-ai/xgrammar v0.1.32,
4// replacing the C++ implementation and the `cxx` FFI bridge. No C/C++
5// /header/Python files; builds with plain `cargo build`.
6//
7// PORT STATUS: COMPLETE (wave W7 — public API + Metrale Engine repoint). The
8// `api` module exposes the exact public surface the vendored
9// `xgrammar-rs` crate provided, so this crate is a drop-in replacement.
10// The algorithmic core lives in the modules below; `api` is the
11// vendored-signature façade Metrale Engine's `metrale-server` links against.
12
13// Index-based loops and same-named submodules (`fsm::fsm`,
14// `compiler::compiler`) are kept where they mirror the C++ source's
15// structure 1:1 — that traceability is deliberate for a line-by-line
16// port and makes diffing against upstream xgrammar tractable.
17#![allow(clippy::needless_range_loop, clippy::module_inception)]
18
19pub mod compiler;
20pub mod earley;
21pub mod fsm;
22pub mod grammar;
23pub mod matcher;
24pub mod regex;
25pub mod schema;
26pub mod structural_tag;
27pub mod support;
28pub mod tokenizer;
29
30mod api;
31
32// ── Public API façade (vendored `xgrammar-rs` surface) ─────────────
33//
34// These are the names Metrale Engine's `use xgrammar::{...}` resolves against.
35// The façade types shadow the core's same-named types at the crate
36// root; the core types remain reachable via their module paths
37// (`xgrammar::compiler::GrammarCompiler`, etc.) for the crate's own
38// tests and any consumer that wants the richer pure-Rust API.
39pub use api::{
40    BatchGrammarMatcher, CompiledGrammar, DLDataType, DLDataTypeCode, DLDevice, DLDeviceType,
41    DLTensor, Grammar, GrammarCompiler, GrammarMatcher, HfMetadata, StructuralTagItem,
42    TokenizerInfo, allocate_token_bitmask, detect_metadata_from_hf, get_bitmask_shape,
43    reset_token_bitmask,
44};
45pub use tokenizer::VocabType;
46
47// ── Core algorithmic exports (richer pure-Rust API) ────────────────
48//
49// Not part of the vendored surface — kept public for the crate's own
50// test suite and downstream consumers that want direct AST access.
51pub use grammar::{GrammarData, GrammarExpr, GrammarExprType, Rule, TagDispatch};
52pub use schema::{
53    JsonFormat, SchemaConverterOptions, SchemaError, deepseek_xml_tool_calling_to_ebnf,
54    json_schema_to_ebnf, json_schema_to_grammar, minimax_xml_tool_calling_to_ebnf,
55    qwen_xml_tool_calling_to_ebnf,
56};
57pub use structural_tag::{
58    StructuralTag, StructuralTagError, structural_tag_from_items, structural_tag_to_grammar,
59};
60
61// ── `VocabType` SCREAMING-CASE aliases ─────────────────────────────
62//
63// The vendored C++ `enum class VocabType` surfaced through autocxx as
64// SCREAMING_SNAKE variants (`VocabType::RAW`, `BYTE_FALLBACK`,
65// `BYTE_LEVEL`). The pure-Rust port follows Rust convention
66// (`Raw`/`ByteFallback`/`ByteLevel`). Metrale Engine's `grammar/engine.rs`
67// writes the SCREAMING form, so we add associated constants that map
68// onto the new variants — both spellings now resolve.
69impl VocabType {
70    /// Vendored alias for [`VocabType::Raw`].
71    pub const RAW: VocabType = VocabType::Raw;
72    /// Vendored alias for [`VocabType::ByteFallback`].
73    pub const BYTE_FALLBACK: VocabType = VocabType::ByteFallback;
74    /// Vendored alias for [`VocabType::ByteLevel`].
75    pub const BYTE_LEVEL: VocabType = VocabType::ByteLevel;
76}
metrale_grammar/lib.rs

metrale_grammar/
lib.rs
