metrale_grammar/structural_tag/
format.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//
3// Structural-tag format AST — port of the `Format` variant family and
4// `StructuralTag` from `cpp/structural_tag.h`.
5//
6// The C++ uses a `std::variant` of ten `*Format` structs plus several
7// `private:` analyzer-only fields (`detected_end_strs_`, `is_unlimited_`).
8// Here that is a single `Format` enum; the analyzer-only fields are
9// public-in-crate so the analyzer pass (`analyzer.rs`) can fill them in,
10// exactly as the C++ `friend class StructuralTagAnalyzer` does.
11
12/// The schema style used to convert a [`Format::JsonSchema`] into EBNF.
13/// Port of `JSONSchemaFormat::style` (`"json"`, `"qwen_xml"`,
14/// `"minimax_xml"`, `"deepseek_xml"`).
15#[derive(Debug, Clone, Copy, PartialEq, Eq)]
16pub enum SchemaStyle {
17    /// Plain JSON schema (`JSONSchemaToEBNF`).
18    Json,
19    /// Qwen XML tool-calling style.
20    QwenXml,
21    /// MiniMax XML tool-calling style.
22    MiniMaxXml,
23    /// DeepSeek XML tool-calling style.
24    DeepSeekXml,
25}
26
27/// One `begin content end` tag. Port of `TagFormat`.
28#[derive(Debug, Clone, PartialEq, Eq)]
29pub struct TagFormat {
30    /// The begin marker, e.g. `<tool_call>`.
31    pub begin: String,
32    /// The tag body format.
33    pub content: Box<Format>,
34    /// Accepted end markers. Multiple markers => any of them ends the
35    /// tag. May be emptied by the analyzer for unlimited content.
36    pub end: Vec<String>,
37}
38
39/// A structural-tag format node. Port of the `Format` `std::variant`.
40#[derive(Debug, Clone, PartialEq, Eq)]
41pub enum Format {
42    /// A fixed literal string. Port of `ConstStringFormat`.
43    ConstString(String),
44    /// Content conforming to a JSON schema (in the given style).
45    /// Port of `JSONSchemaFormat`.
46    JsonSchema {
47        /// The JSON-schema document, serialized as a JSON string.
48        json_schema: String,
49        /// Conversion style.
50        style: SchemaStyle,
51    },
52    /// Arbitrary text, optionally excluding some substrings. Port of
53    /// `AnyTextFormat`.
54    AnyText {
55        /// Substrings that must not appear in the matched text.
56        excludes: Vec<String>,
57        /// End strings detected by the analyzer from an enclosing tag.
58        detected_end_strs: Vec<String>,
59    },
60    /// Content conforming to a raw EBNF grammar. Port of `GrammarFormat`.
61    Grammar(String),
62    /// Content conforming to a regex pattern. Port of `RegexFormat`.
63    Regex(String),
64    /// An ordered concatenation of sub-formats. Port of `SequenceFormat`.
65    Sequence {
66        /// The element formats.
67        elements: Vec<Format>,
68        /// Set by the analyzer: true if the last element is unlimited.
69        is_unlimited: bool,
70    },
71    /// An alternation of sub-formats. Port of `OrFormat`.
72    Or {
73        /// The alternative formats.
74        elements: Vec<Format>,
75        /// Set by the analyzer: true if (all) elements are unlimited.
76        is_unlimited: bool,
77    },
78    /// A single tag. Port of `TagFormat`.
79    Tag(TagFormat),
80    /// Triggered tag dispatch — the tool-call envelope mechanism.
81    /// Port of `TriggeredTagsFormat`.
82    TriggeredTags {
83        /// Trigger prefixes; each tag's `begin` starts with one trigger.
84        triggers: Vec<String>,
85        /// The tags dispatched to.
86        tags: Vec<TagFormat>,
87        /// Substrings excluded from inter-tag free text.
88        excludes: Vec<String>,
89        /// If true, at least one tag must be produced.
90        at_least_one: bool,
91        /// If true, generation stops after the first tag.
92        stop_after_first: bool,
93        /// End strings detected by the analyzer from an enclosing tag.
94        detected_end_strs: Vec<String>,
95    },
96    /// Tags joined by a separator. Port of `TagsWithSeparatorFormat`.
97    TagsWithSeparator {
98        /// The tags.
99        tags: Vec<TagFormat>,
100        /// The separator string between tags.
101        separator: String,
102        /// If true, at least one tag must be produced.
103        at_least_one: bool,
104        /// If true, generation stops after the first tag.
105        stop_after_first: bool,
106        /// End strings detected by the analyzer from an enclosing tag.
107        detected_end_strs: Vec<String>,
108    },
109}
110
111/// The top-level structural tag. Port of `StructuralTag`.
112#[derive(Debug, Clone, PartialEq, Eq)]
113pub struct StructuralTag {
114    /// The root format.
115    pub format: Format,
116}
117
118/// A single legacy structural-tag item: `begin`, a JSON `schema`
119/// string, and `end`. Port of the public `StructuralTagItem` struct
120/// from `include/xgrammar/grammar.h`.
121///
122/// This is the tool-calling envelope descriptor: each item says
123/// "a tool call starts with `begin`, its arguments conform to
124/// `schema`, and it ends with `end`".
125#[derive(Debug, Clone, PartialEq, Eq)]
126pub struct StructuralTagItem {
127    /// The begin marker of the tag, e.g. `<function=get_weather>`.
128    pub begin: String,
129    /// The JSON-schema document (as a string) for the tag body.
130    pub schema: String,
131    /// The end marker of the tag, e.g. `</function>`.
132    pub end: String,
133}
134
135impl StructuralTagItem {
136    /// Construct a [`StructuralTagItem`].
137    pub fn new(
138        begin: impl Into<String>,
139        schema: impl Into<String>,
140        end: impl Into<String>,
141    ) -> Self {
142        Self {
143            begin: begin.into(),
144            schema: schema.into(),
145            end: end.into(),
146        }
147    }
148}
metrale_grammar/structural_tag/format.rs

metrale_grammar/structural_tag/
format.rs
