metrale_bench/benchmarks/
transcript.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: What a request emitted, reduced to the parts the equality gates
4//! compare (contamination, scheduler equivalence, KAT equality, SSM poisoning).
5//!
6//! Owner: bench.
7//! Invariants:
8//! - `canonical` holds reasoning, text, tool calls and finish reason, and no
9//!   token count.
10
11use crate::http::ChatOutcome;
12
13/// 2026-09-26: The comparable part of one reply.
14#[derive(Debug, Clone, PartialEq, Eq, Default)]
15pub struct Transcript {
16    /// 2026-09-26: Kept apart from `text`, as `ChatOutcome` does, so a change of
17    /// reasoning is not hidden by an identical answer.
18    pub reasoning: String,
19    pub text: String,
20    /// 2026-09-26: `(name, raw arguments)`. Arguments stay raw: re-serialising
21    /// the JSON would normalise key order and whitespace.
22    pub tool_calls: Vec<(String, String)>,
23    pub finish_reason: Option<String>,
24    /// 2026-09-26: `ChatOutcome::completion_tokens`: the server's
25    /// `usage.completion_tokens` when the stream carries it, else the streamed
26    /// delta count.
27    pub completion_tokens: usize,
28    /// 2026-09-26: Diagnostic: not in `canonical`, so no divergence check here
29    /// compares it. The SSM poisoning gate reads it as its turn-1 cache count.
30    pub cached_prompt_tokens: usize,
31}
32
33impl From<&ChatOutcome> for Transcript {
34    fn from(o: &ChatOutcome) -> Self {
35        Self {
36            reasoning: o.reasoning.clone(),
37            text: o.text.clone(),
38            tool_calls: o
39                .tool_calls
40                .iter()
41                .map(|t| (t.name.clone(), t.arguments.clone()))
42                .collect(),
43            finish_reason: o.finish_reason.clone(),
44            completion_tokens: o.completion_tokens,
45            cached_prompt_tokens: o.cached_prompt_tokens,
46        }
47    }
48}
49
50impl Transcript {
51    /// 2026-09-26: The text a divergence check compares, concatenated in a fixed
52    /// order with control-character separators; callers also use it for
53    /// longest-common-prefix localisation. The token counts are left out:
54    /// callers compare `completion_tokens` separately.
55    pub fn canonical(&self) -> String {
56        let mut s = String::with_capacity(self.reasoning.len() + self.text.len() + 64);
57        s.push_str(&self.reasoning);
58        s.push('\u{1}');
59        s.push_str(&self.text);
60        for (name, args) in &self.tool_calls {
61            s.push('\u{2}');
62            s.push_str(name);
63            s.push('\u{3}');
64            s.push_str(args);
65        }
66        s.push('\u{4}');
67        s.push_str(self.finish_reason.as_deref().unwrap_or(""));
68        s
69    }
70
71    /// 2026-09-26: The first canary in `all`, other than `own`, that appears in
72    /// this reply's canonical text. Unlike a diff, it needs no reference.
73    pub fn carries_foreign_canary<'a>(&self, own: &str, all: &[&'a str]) -> Option<&'a str> {
74        let hay = self.canonical();
75        all.iter().find(|c| **c != own && hay.contains(*c)).copied()
76    }
77}
78
79/// 2026-09-26: One request's result. An error is its own variant, so callers
80/// score it apart from a transcript.
81#[derive(Debug, Clone, PartialEq, Eq)]
82pub enum RequestOutcome {
83    Ok(Box<Transcript>),
84    Error(String),
85}
86
87impl RequestOutcome {
88    pub fn transcript(&self) -> Option<&Transcript> {
89        match self {
90            Self::Ok(t) => Some(t),
91            Self::Error(_) => None,
92        }
93    }
94}
metrale_bench/benchmarks/transcript.rs

metrale_bench/benchmarks/
transcript.rs
