metrale_grammar/tokenizer/
mod.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Tokenizer metadata for the grammar matcher: `TokenizerInfo` (the decoded
4//! vocabulary), `VocabType` (`Raw`, `ByteFallback`, `ByteLevel`), `decode_token` and the
5//! byte-level character maps, and `detect_metadata_from_hf`, which reads the vocabulary
6//! type and prefix-space flag from the text of a `tokenizer.json`.
7//!
8//! Owner: grammar.
9//! Invariants: none beyond the types.
10
11pub mod decoder;
12pub mod hf_metadata;
13pub mod info;
14pub mod vocab_type;
15
16pub use decoder::{byte_to_char_map, char_to_byte_map, decode_token};
17pub use hf_metadata::{HfMetadata, detect_metadata_from_hf};
18pub use info::TokenizerInfo;
19pub use vocab_type::VocabType;