metrale_bench/benchmarks/ssm_poison/
toolcall.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Path-independence of tool calls: one target turn, whose correct
4//! answer is to call nothing, reached by three predecessor paths that share the
5//! same long prefix. The replay script offers no tools, and its comparison
6//! passes reworded prose, so it cannot see a tool call that moves with history.
7//!
8//! Owner: bench, SSM poisoning gate.
9//! Invariants:
10//! - Only the target's tool calls are compared, raw and in order; its prose is
11//!   not.
12//!
13//! ```text
14//! A  prefix → ack → TARGET
15//! B  prefix → ack → a turn that calls get_current_weather → TARGET
16//! C  prefix → ack → a turn that calls web_search          → TARGET
17//! ```
18//!
19//! The user turns of the paths differ only in the interposed turn. Path A is
20//! the reference; the target's `tool_calls` must equal A's on B and C. The
21//! target asks for something no offered tool answers, so a call there is wrong
22//! whatever its wording.
23
24use serde_json::{Value, json};
25
26use crate::benchmarks::transcript::Transcript;
27
28/// 2026-09-26: The tool schemas offered on every turn of this probe.
29pub fn tools() -> Vec<Value> {
30    vec![
31        json!({
32            "type": "function",
33            "function": {
34                "name": "get_current_weather",
35                "description": "Get the current weather for a location.",
36                "parameters": {
37                    "type": "object",
38                    "properties": {
39                        "location": {"type": "string", "description": "City and state, e.g. San Francisco, CA"}
40                    },
41                    "required": ["location"]
42                }
43            }
44        }),
45        json!({
46            "type": "function",
47            "function": {
48                "name": "web_search",
49                "description": "Search the public web for a query string.",
50                "parameters": {
51                    "type": "object",
52                    "properties": {
53                        "query": {"type": "string", "description": "The search query"}
54                    },
55                    "required": ["query"]
56                }
57            }
58        }),
59    ]
60}
61
62/// 2026-09-26: The interposed turn of path B, which asks for a
63/// `get_current_weather` call.
64pub const CALLS_WEATHER: &str = "What is the current weather in San Francisco, CA? Use the tool.";
65
66/// 2026-09-26: The interposed turn of path C, which asks for a `web_search` call.
67pub const CALLS_SEARCH: &str =
68    "Search the web for the VirusTotal API domain report endpoint. Use the tool.";
69
70/// 2026-09-26: The target turn. Its answer is in the document the conversation
71/// already carries, and neither offered tool bears on it, so the correct reply
72/// calls nothing.
73pub const TARGET: &str = "From SYSTEM DOCUMENT 7741-C already in this \
74     conversation, state in one sentence what closed membership means. Answer \
75     from the document itself.";
76
77/// 2026-09-26: A labelled predecessor path to [`TARGET`].
78#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79pub enum Path {
80    /// 2026-09-26: prefix → ack → target. The reference.
81    Direct,
82    /// 2026-09-26: prefix → ack → [`CALLS_WEATHER`] → target.
83    AfterWeather,
84    /// 2026-09-26: prefix → ack → [`CALLS_SEARCH`] → target.
85    AfterSearch,
86}
87
88impl Path {
89    pub const ALL: [Path; 3] = [Path::Direct, Path::AfterWeather, Path::AfterSearch];
90
91    pub fn label(self) -> &'static str {
92        match self {
93            Path::Direct => "direct",
94            Path::AfterWeather => "after-weather",
95            Path::AfterSearch => "after-search",
96        }
97    }
98
99    /// 2026-09-26: The turn to interpose between the ack and the target, if any.
100    pub fn interposed(self) -> Option<&'static str> {
101        match self {
102            Path::Direct => None,
103            Path::AfterWeather => Some(CALLS_WEATHER),
104            Path::AfterSearch => Some(CALLS_SEARCH),
105        }
106    }
107}
108
109/// 2026-09-26: The request body for a tool-offering turn: the replay probe's
110/// greedy, seed-0 body plus [`tools`] and `tool_choice: "auto"`, so the model
111/// decides whether to call.
112pub fn request_body(model: &str, messages: &[Value], max_tokens: usize) -> Value {
113    json!({
114        "model": model,
115        "stream": true,
116        "stream_options": {"include_usage": true},
117        "temperature": 0.0,
118        "seed": 0,
119        "max_tokens": max_tokens,
120        "messages": messages,
121        "tools": tools(),
122        "tool_choice": "auto",
123    })
124}
125
126/// 2026-09-26: One path's outcome: the target turn's transcript.
127#[derive(Debug, Clone)]
128pub struct PathResult {
129    pub path: Path,
130    pub target: Transcript,
131}
132
133impl PathResult {
134    /// 2026-09-26: The comparison key: the tool calls alone, as raw
135    /// `(name, arguments)` pairs in order. Prose is excluded.
136    pub fn calls(&self) -> Vec<(String, String)> {
137        self.target.tool_calls.clone()
138    }
139}
140
141/// 2026-09-26: One path disagreeing with the reference.
142#[derive(Debug, Clone, PartialEq, Eq)]
143pub struct Divergence {
144    pub path: &'static str,
145    pub reference_calls: Vec<(String, String)>,
146    pub path_calls: Vec<(String, String)>,
147}
148
149impl Divergence {
150    /// 2026-09-26: A one-line reading for the log and the verdict.
151    pub fn describe(&self) -> String {
152        let fmt = |c: &Vec<(String, String)>| {
153            if c.is_empty() {
154                "no call".to_string()
155            } else {
156                c.iter()
157                    .map(|(n, a)| format!("{n}({a})"))
158                    .collect::<Vec<_>>()
159                    .join(" + ")
160            }
161        };
162        format!(
163            "{}: reference made {}, this path made {}",
164            self.path,
165            fmt(&self.reference_calls),
166            fmt(&self.path_calls)
167        )
168    }
169}
170
171/// 2026-09-26: Compare every path against the reference (`Path::Direct`).
172/// Returns one [`Divergence`] per path whose tool calls differ at all; there is
173/// no tolerance band as there is for replay prose. Empty when every path
174/// agrees, and also when no reference path is present.
175pub fn divergences(results: &[PathResult]) -> Vec<Divergence> {
176    let Some(reference) = results.iter().find(|r| r.path == Path::Direct) else {
177        return Vec::new();
178    };
179    let ref_calls = reference.calls();
180    results
181        .iter()
182        .filter(|r| r.path != Path::Direct)
183        .filter_map(|r| {
184            let calls = r.calls();
185            (calls != ref_calls).then(|| Divergence {
186                path: r.path.label(),
187                reference_calls: ref_calls.clone(),
188                path_calls: calls,
189            })
190        })
191        .collect()
192}
193
194/// 2026-09-26: Did the reference path itself call a tool on the target? A
195/// different finding from [`divergences`]: the driver fails the run on a
196/// divergence and only logs this one.
197pub fn reference_called(results: &[PathResult]) -> bool {
198    results
199        .iter()
200        .find(|r| r.path == Path::Direct)
201        .is_some_and(|r| !r.calls().is_empty())
202}
203
204#[cfg(test)]
205#[path = "toolcall_tests.rs"]
206mod tests;
metrale_bench/benchmarks/ssm_poison/toolcall.rs

metrale_bench/benchmarks/ssm_poison/
toolcall.rs
