metrale_grammar/compiler/
mod.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Grammar compiler: turns a grammar, a JSON schema or a structural tag,
4//! together with a tokenizer, into a `CompiledGrammar`, which holds the grammar and
5//! computes, per parser state, the token masks the matcher uses to fill the logit bitmask.
6//!
7//! `GrammarCompiler` caches compiled grammars in `GrammarCache`, and, when caching is
8//! enabled, shares one `RuleLevelCache` of per-rule masks across every grammar it builds.
9//!
10//! Owner: grammar.
11//! Invariants: none beyond the types.
12
13mod coalesce;
14mod compile;
15mod compiled_grammar;
16mod compiler;
17mod decompose;
18mod grammar_cache;
19mod mask;
20mod mask_gen;
21pub mod mask_snapshot;
22mod prewarm;
23mod rule_cache;
24
25pub use coalesce::{Forced, analyze_bitmask};
26pub use compiled_grammar::{CompiledGrammar, CompiledGrammarImpl};
27pub use compiler::{CompileError, GrammarCompiler};
28pub use decompose::{GrammarDecomposition, RuleDecomposition, Segment, decompose_static_regions};
29pub use mask::{AdaptiveTokenMask, StoreType, USE_BITSET_THRESHOLD};
30pub use mask_snapshot::{SnapshotError, SnapshotIdentity};
31pub use rule_cache::{RuleLevelCache, RuleMaskKey, UNLIMITED_SIZE};
32
33#[cfg(test)]
34mod tests;
metrale_grammar/compiler/mod.rs

metrale_grammar/compiler/
mod.rs
