metrale_bench/benchmarks/agentic/
trace.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Per-iteration trajectory file: the prompt, then each turn's
4//! reasoning, text, tool calls and finish reason, and each tool result, as
5//! plain text so `diff` of two runs shows the first turn at which they split.
6//! The outcome columns (`webserver_ok`, steps, turns) can agree while the
7//! trajectories differ.
8//!
9//! Owner: bench, agentic.
10//! Invariants:
11//! - The file is written beside the sandbox (`run-07` gives
12//!   `run-07.trajectory.txt`), never inside it: the agent's `read`, `glob` and
13//!   `grep` tools and the scorer's `has_tests` walk the sandbox.
14//! - A write failure is logged and never returned as an error.
15
16use std::io::Write;
17use std::path::{Path, PathBuf};
18
19use crate::http::ChatOutcome;
20
21pub struct Trace {
22    path: Option<PathBuf>,
23}
24
25impl Trace {
26    /// 2026-09-26: Truncates a trace left by an earlier run of the same index,
27    /// so the file holds one run.
28    pub fn start(sandbox: &Path, prompt: &str) -> Self {
29        let path = path_for(sandbox);
30        let trace = Self { path: Some(path) };
31        match std::fs::write(trace.path.as_ref().expect("just set"), header(prompt)) {
32            Ok(()) => trace,
33            Err(e) => {
34                tracing::warn!("agentic: trajectory trace disabled: {e}");
35                Self { path: None }
36            }
37        }
38    }
39
40    /// 2026-09-26: One model turn: its reasoning, its reply, the calls it asked
41    /// for and its finish reason. Empty reasoning or text sections are omitted.
42    pub fn turn(&self, index: usize, outcome: &ChatOutcome) {
43        let mut s = format!("\n── turn {} ───────────────────────────────\n", index + 1);
44        section(&mut s, "reasoning", &outcome.reasoning);
45        section(&mut s, "text", &outcome.text);
46        for call in &outcome.tool_calls {
47            s.push_str(&format!(
48                "[call {}] {} {}\n",
49                call.id, call.name, call.arguments
50            ));
51        }
52        if let Some(reason) = &outcome.finish_reason {
53            s.push_str(&format!("[finish] {reason}\n"));
54        }
55        self.append(&s);
56    }
57
58    /// 2026-09-26: One tool result, as the content sent back to the model
59    /// (truncated, and for shell output normalised), with trailing whitespace
60    /// trimmed.
61    pub fn result(&self, tool: &str, content: &str) {
62        let mut s = format!("[result {tool}]\n");
63        s.push_str(content.trim_end());
64        s.push('\n');
65        self.append(&s);
66    }
67
68    fn append(&self, text: &str) {
69        let Some(path) = &self.path else { return };
70        let written = std::fs::OpenOptions::new()
71            .append(true)
72            .open(path)
73            .and_then(|mut f| f.write_all(text.as_bytes()));
74        if let Err(e) = written {
75            tracing::warn!("agentic: could not extend {}: {e}", path.display());
76        }
77    }
78}
79
80fn path_for(sandbox: &Path) -> PathBuf {
81    let mut name = sandbox.file_name().unwrap_or_default().to_os_string();
82    name.push(".trajectory.txt");
83    sandbox.with_file_name(name)
84}
85
86fn header(prompt: &str) -> String {
87    format!("[prompt]\n{}\n", prompt.trim_end())
88}
89
90fn section(out: &mut String, name: &str, body: &str) {
91    if body.trim().is_empty() {
92        return;
93    }
94    out.push_str(&format!("[{name}]\n{}\n", body.trim_end()));
95}
96
97#[cfg(test)]
98#[path = "trace_tests.rs"]
99mod tests;
metrale_bench/benchmarks/agentic/trace.rs

metrale_bench/benchmarks/agentic/
trace.rs
