metrale_grammar/structural_tag/
mod.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//
3// Structural tag — pure-Rust port of `cpp/structural_tag.{cc,h}`.
4//
5// WHAT THIS IS
6// ------------
7// The structural tag is the tool-calling envelope mechanism. A model
8// is constrained to emit a stream where, between *trigger* points,
9// free text is allowed, and once a trigger prefix appears the model
10// must produce a tagged body — `begin <schema-conforming content> end`
11// — i.e. `<tool_call>{json}</tool_call>` style enforcement. Metrale Engine's
12// `grammar/compile_tools.rs` calls into this for every tool request.
13//
14// PIPELINE (mirrors `StructuralTagToGrammar`)
15// -------------------------------------------
16//   1. `parser`      — structural-tag JSON document -> `StructuralTag`
17//                      AST (`Format` tree).
18//   2. `analyzer`    — fills analyzer-only fields: detected end strings
19//                      and the `is_unlimited` flags.
20//   3. `converter`   — `Format` tree -> BNF rules via `GrammarBuilder`,
21//                      reusing `src/schema` for embedded JSON schemas
22//                      and `src/regex` for regex content.
23//   4. normalize     — `GrammarNormalizer` runs as the final step.
24//
25// FAITHFULNESS / SIMPLIFICATIONS vs C++
26// -------------------------------------
27//  * C++ returns `Result<Grammar, StructuralTagError>` where the error
28//    is a `std::variant`; we return `Result<_, StructuralTagError>`
29//    with a two-variant enum carrying the same JSON-vs-tag distinction.
30//  * C++ `RecursionGuard` aborts on overflow; we return an error.
31//  * The C++ `StructuralTagAnalyzer` keeps a raw-pointer stack to find
32//    the enclosing tag's end strings; we thread that list down the
33//    recursion explicitly — equivalent, zero `unsafe`.
34//  * Embedded `json_schema` sub-documents are re-serialized with an
35//    order-preserving serializer (`json_serialize`) so JSON-Schema
36//    property order survives, matching picojson `serialize(false)`.
37
38mod analyzer;
39mod converter;
40mod converter_sep;
41mod converter_tags;
42mod error;
43mod format;
44mod json_serialize;
45mod parser;
46mod parser_tags;
47
48#[cfg(test)]
49mod tests;
50
51pub use error::{StructuralTagError, StructuralTagResult};
52pub use format::{Format, SchemaStyle, StructuralTag, StructuralTagItem, TagFormat};
53
54use crate::grammar::GrammarData;
55use crate::grammar::builder::TagDispatchSpec;
56
57/// Build a [`TagDispatchSpec`] from the structural-tag fields. Shared
58/// by the `any_text` and `triggered_tags` converters.
59pub(crate) fn tag_dispatch_spec(
60    tag_rule_pairs: Vec<(String, i32)>,
61    stop_eos: bool,
62    stop_str: Vec<String>,
63    loop_after_dispatch: bool,
64    excluded_str: Vec<String>,
65) -> TagDispatchSpec {
66    TagDispatchSpec {
67        tag_rule_pairs,
68        stop_eos,
69        stop_str,
70        loop_after_dispatch,
71        excluded_str,
72    }
73}
74
75/// Convert a structural-tag JSON document into a grammar.
76///
77/// Port of `StructuralTagToGrammar` / `Grammar::FromStructuralTag(json)`.
78/// The document must be a `{"type": "structural_tag", "format": {...}}`
79/// object (the `type` field is optional but checked when present).
80pub fn structural_tag_to_grammar(structural_tag_json: &str) -> StructuralTagResult<GrammarData> {
81    let mut structural_tag = parser::StructuralTagParser::from_json(structural_tag_json)?;
82    analyzer::analyze(&mut structural_tag)?;
83    converter::StructuralTagConverter::convert(&structural_tag)
84}
85
86/// Build a grammar from a list of [`StructuralTagItem`]s and trigger
87/// prefixes — the legacy `Grammar::FromStructuralTag(tags, triggers)`
88/// entry point.
89///
90/// Each item becomes a `tag` (`begin`, a `json_schema` body, `end`)
91/// inside a single `triggered_tags` format whose triggers are
92/// `triggers`. This is the exact shape Metrale Engine's tool-call compiler
93/// uses: one tag per tool, one trigger per opening marker.
94///
95/// Port of `StructuralTag::from_legacy_structural_tag` followed by
96/// `StructuralTagToGrammar`.
97pub fn structural_tag_from_items(
98    tags: &[StructuralTagItem],
99    triggers: &[String],
100) -> StructuralTagResult<GrammarData> {
101    let json = build_legacy_json(tags, triggers)?;
102    structural_tag_to_grammar(&json)
103}
104
105/// Assemble the structural-tag JSON document for the legacy
106/// `(tags, triggers)` form. Each tag's `schema` string is embedded
107/// verbatim as the `json_schema` value (it must itself be valid JSON).
108fn build_legacy_json(
109    tags: &[StructuralTagItem],
110    triggers: &[String],
111) -> StructuralTagResult<String> {
112    let mut tag_objs = Vec::with_capacity(tags.len());
113    for item in tags {
114        // Validate the schema parses as JSON before embedding it.
115        crate::schema::JsonValue::parse(&item.schema).map_err(|e| {
116            StructuralTagError::json(format!("structural tag item schema is not valid JSON: {e}"))
117        })?;
118        tag_objs.push(format!(
119            r#"{{"type":"tag","begin":{},"content":{{"type":"json_schema","json_schema":{}}},"end":{}}}"#,
120            json_string(&item.begin),
121            item.schema,
122            json_string(&item.end),
123        ));
124    }
125    let trigger_strs: Vec<String> = triggers.iter().map(|t| json_string(t)).collect();
126    Ok(format!(
127        r#"{{"type":"structural_tag","format":{{"type":"triggered_tags","triggers":[{}],"tags":[{}]}}}}"#,
128        trigger_strs.join(","),
129        tag_objs.join(","),
130    ))
131}
132
133/// Render `s` as a JSON string literal.
134fn json_string(s: &str) -> String {
135    let mut out = String::with_capacity(s.len() + 2);
136    out.push('"');
137    for c in s.chars() {
138        match c {
139            '"' => out.push_str("\\\""),
140            '\\' => out.push_str("\\\\"),
141            '\n' => out.push_str("\\n"),
142            '\r' => out.push_str("\\r"),
143            '\t' => out.push_str("\\t"),
144            '\u{08}' => out.push_str("\\b"),
145            '\u{0c}' => out.push_str("\\f"),
146            c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)),
147            c => out.push(c),
148        }
149    }
150    out.push('"');
151    out
152}
metrale_grammar/structural_tag/mod.rs

metrale_grammar/structural_tag/
mod.rs
