metrale_bench/benchmarks/ssm_poison/
compare.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Round comparison: every replay round against the reference round.
4//!
5//! Owner: bench, SSM poisoning gate.
6//! Invariants:
7//! - A round is `Invariant` only when every turn's canonical transcript and
8//!   token count equal the reference turn's.
9//! - A differing turn is collapsed when its finish reason differs or its token
10//!   ratio leaves [`COLLAPSE_RATIO_FLOOR`]..=[`COLLAPSE_RATIO_CEIL`], and
11//!   jittered otherwise.
12//! - A round's verdict is `Collapsed` if any turn collapsed, else `Unmeasured`
13//!   if any turn was, else `Jittered` if any turn jittered.
14
15use crate::benchmarks::transcript::Transcript;
16
17/// 2026-09-26: A replay turn shorter than this fraction of the reference turn,
18/// in completion tokens, is a collapse.
19pub const COLLAPSE_RATIO_FLOOR: f64 = 0.5;
20/// 2026-09-26: A replay turn longer than this multiple of the reference turn is a
21/// collapse too.
22pub const COLLAPSE_RATIO_CEIL: f64 = 2.0;
23
24/// 2026-09-26: One differing turn's shape.
25#[derive(Debug, Clone, PartialEq, Eq)]
26pub struct TurnDelta {
27    pub turn: usize,
28    pub ref_tokens: usize,
29    pub replay_tokens: usize,
30    pub ref_finish: Option<String>,
31    pub replay_finish: Option<String>,
32}
33
34impl TurnDelta {
35    /// 2026-09-26: A different finish reason, or a token ratio outside the
36    /// collapse window.
37    pub fn is_collapse(&self) -> bool {
38        if self.ref_finish != self.replay_finish {
39            return true;
40        }
41        if self.ref_tokens == 0 {
42            // 2026-09-26: A nonzero replay against a zero-token reference is an
43            // infinite ratio, so a collapse. The both-zero pair is not;
44            // `compare_round` scores that turn Unmeasured before asking.
45            return self.replay_tokens != 0;
46        }
47        let ratio = self.replay_tokens as f64 / self.ref_tokens as f64;
48        !(COLLAPSE_RATIO_FLOOR..=COLLAPSE_RATIO_CEIL).contains(&ratio)
49    }
50}
51
52/// 2026-09-26: The outcome of comparing one replay round to the reference round.
53#[derive(Debug, Clone, PartialEq, Eq)]
54pub enum RoundVerdict {
55    /// 2026-09-26: Every turn equal: canonical transcript and token count.
56    Invariant,
57    /// 2026-09-26: At least one turn differs, and every differing turn has the
58    /// same finish reason and a token ratio inside the window. Recorded; the
59    /// verdict passes it.
60    Jittered { turns: Vec<TurnDelta> },
61    /// 2026-09-26: At least one turn collapsed (`TurnDelta::is_collapse`).
62    Collapsed { turns: Vec<TurnDelta> },
63    /// 2026-09-26: The round proves nothing: the replay errored (set by the
64    /// driver), its turn count differs from the reference's, or a turn
65    /// returned no tokens in both rounds.
66    Unmeasured { reason: String },
67}
68
69/// 2026-09-26: Compare a reference turn list against a replay turn list. A replay
70/// whose turn count differs from the reference's is Unmeasured, never
71/// Invariant.
72pub fn compare_round(reference: &[Transcript], replay: &[Transcript]) -> RoundVerdict {
73    if replay.len() != reference.len() {
74        return RoundVerdict::Unmeasured {
75            reason: format!(
76                "replay produced {} turn(s), reference has {}",
77                replay.len(),
78                reference.len()
79            ),
80        };
81    }
82    if reference.is_empty() {
83        return RoundVerdict::Unmeasured {
84            reason: "reference round has no turns".into(),
85        };
86    }
87    let mut jittered = Vec::new();
88    let mut collapsed = Vec::new();
89    let mut unmeasured: Option<String> = None;
90    for (i, (r, p)) in reference.iter().zip(replay).enumerate() {
91        if r.completion_tokens == 0 && p.completion_tokens == 0 {
92            // 2026-09-26: Two empty replies are equal and prove nothing.
93            unmeasured = Some(format!("turn {} returned no tokens", i + 1));
94            continue;
95        }
96        if r.canonical() == p.canonical() && r.completion_tokens == p.completion_tokens {
97            continue;
98        }
99        let delta = TurnDelta {
100            turn: i + 1,
101            ref_tokens: r.completion_tokens,
102            replay_tokens: p.completion_tokens,
103            ref_finish: r.finish_reason.clone(),
104            replay_finish: p.finish_reason.clone(),
105        };
106        if delta.is_collapse() {
107            collapsed.push(delta);
108        } else {
109            jittered.push(delta);
110        }
111    }
112    if !collapsed.is_empty() {
113        return RoundVerdict::Collapsed { turns: collapsed };
114    }
115    // 2026-09-26: Unmeasured outranks Jittered, because the verdict fails an
116    // unmeasured round and passes a jittered one. Collapsed, checked above,
117    // is the more specific failure.
118    if let Some(reason) = unmeasured {
119        return RoundVerdict::Unmeasured { reason };
120    }
121    if !jittered.is_empty() {
122        return RoundVerdict::Jittered { turns: jittered };
123    }
124    RoundVerdict::Invariant
125}
126
127#[cfg(test)]
128#[path = "compare_tests.rs"]
129mod compare_tests;
metrale_bench/benchmarks/ssm_poison/compare.rs

metrale_bench/benchmarks/ssm_poison/
compare.rs
