metrale_bench/benchmarks/agentic/
mod.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The `agentic-webserver` benchmark: N iterations of one task
4//! (write a Rust Axum ping/pong project, test it, run it, curl it, tear it
5//! down), each in a fresh sandbox. After each iteration the scorer builds and
6//! runs what the agent left and asks `/ping` for `pong`, and checks the agent's
7//! commands against the prompt's directions.
8//!
9//! The prompt is verbatim from `bench/fp8_dgx2_drift/harness/run_tier.sh`; the
10//! agent loop is our own, not the opencode client that script drives. It runs
11//! model-authored shell; see [`agent`] for the containment.
12//!
13//! Owner: bench, agentic.
14//! Invariants: none beyond the types.
15
16pub mod agent;
17mod params;
18pub mod preflight;
19mod render;
20pub mod score;
21mod verdict;
22pub mod warm;
23
24use std::future::Future;
25use std::path::PathBuf;
26use std::time::{Duration, Instant};
27
28use anyhow::{Context, Result, bail};
29
30use crate::benchmark::{Benchmark, BenchmarkDescriptor};
31use crate::benchmarks::one_line;
32use crate::http;
33use crate::metadata::PluginMetadata;
34use crate::params::{ParamSpec, ParamValues};
35use crate::plugin::{Plugin, PluginHandle};
36use crate::result::{BenchmarkResult, LogLine, RunStatus, Verdict};
37
38/// 2026-09-26: The task, verbatim from `run_tier.sh`'s `PROMPT`. Changing a word
39/// changes the benchmark.
40pub const PROMPT: &str = "Please create a pure rust Axum project here in the current working \
41directory. Just have a ping/pong endpoint. The server MUST bind to the port from the \
42METRALE_HARNESS_PORT env var (default 3001) — use `let port: u16 = \
43std::env::var(\"METRALE_HARNESS_PORT\").unwrap_or_else(|_| \"3001\".to_string()).parse().unwrap();` \
44then bind to `0.0.0.0:port`. Add tests, run them and prove all tests pass, then run the server and \
45use curl to prove it works. Whenever you run the server or any long-lived process in the \
46background, always start it detached with its output redirected to a file (for example `setsid \
47cargo run > /tmp/server.log 2>&1 &`) so your shell never blocks waiting on it, and wrap any \
48command that might hang, such as curl checks or process kills, in a short `timeout 15`. Finally, \
49tear down the server by killing whatever is listening on its port rather than guessing the process \
50name, always wrapped in a short timeout so it can never stall your shell, for example `timeout 5 \
51fuser -k ${METRALE_HARNESS_PORT:-3001}/tcp 2>/dev/null || true`.";
52
53mod descriptors;
54pub use descriptors::{DESCRIPTOR, METADATA};
55
56#[derive(Default)]
57struct IterationRow {
58    index: usize,
59    /// 2026-09-26: Iteration wall including the scorer's build and probe;
60    /// `sum_wall_s` sums it.
61    wall_s: f64,
62    /// 2026-09-26: The agent's own wall, scorer excluded. `s_per_turn` divides
63    /// its sum by turns; the scorer is a per-iteration cost, so it is left out
64    /// of a per-turn ratio.
65    agent_wall_s: f64,
66    webserver_ok: bool,
67    directions: score::Directions,
68    turns: usize,
69    tool_calls: usize,
70    completion_tokens: usize,
71    /// 2026-09-26: The loop ended at `max_turns`, not because the agent stopped
72    /// calling tools.
73    hit_turn_cap: bool,
74    /// 2026-09-26: See `Transcript::truncated_turns`.
75    truncated_turns: usize,
76    /// 2026-09-26: See `Transcript::unparsed_call_turns`.
77    unparsed_call_turns: usize,
78    note: String,
79}
80
81#[derive(Default)]
82pub struct AgenticWebserver {
83    handle: Option<PluginHandle>,
84    iterations: usize,
85    max_turns: usize,
86    command_timeout: Duration,
87    request_timeout: Duration,
88    build_timeout: Duration,
89    serve_timeout: Duration,
90    max_tokens: usize,
91    wall_budget_s: f64,
92    s_per_turn_budget: f64,
93    cursor: usize,
94    rows: Vec<IterationRow>,
95    sandbox_root: Option<PathBuf>,
96    cargo_target_dir: Option<PathBuf>,
97    started: Option<Instant>,
98    probed: bool,
99}
100
101impl AgenticWebserver {
102    fn handle(&self) -> Result<&PluginHandle> {
103        self.handle.as_ref().context("benchmark was not loaded")
104    }
105
106    fn elapsed(&self) -> Duration {
107        self.started.map(|s| s.elapsed()).unwrap_or_default()
108    }
109
110    fn total_wall(&self) -> f64 {
111        self.rows.iter().map(|r| r.wall_s).sum()
112    }
113
114    /// 2026-09-26: Sum of `agent_wall_s`, the numerator of `s_per_turn`.
115    fn total_agent_wall(&self) -> f64 {
116        self.rows.iter().map(|r| r.agent_wall_s).sum()
117    }
118
119    fn total_turns(&self) -> usize {
120        verdict::total_turns(&self.rows)
121    }
122
123    fn total_tool_calls(&self) -> usize {
124        self.rows.iter().map(|r| r.tool_calls).sum()
125    }
126
127    fn total_completion_tokens(&self) -> usize {
128        self.rows.iter().map(|r| r.completion_tokens).sum()
129    }
130
131    /// 2026-09-26: `None` when the tier took no turns ([`verdict::seconds_per_turn`]).
132    fn seconds_per_turn(&self) -> Option<f64> {
133        verdict::seconds_per_turn(self.total_agent_wall(), self.total_turns())
134    }
135
136    async fn run_iteration(&mut self, index: usize) -> Result<IterationRow> {
137        let handle = self.handle()?.clone();
138        let root = self
139            .sandbox_root
140            .clone()
141            .context("sandbox root was not prepared")?;
142        let sandbox = root.join(format!("run-{index:02}"));
143        // 2026-09-26: A fresh directory per iteration, so an agent is never
144        // scored on code a previous run left.
145        let _ = std::fs::remove_dir_all(&sandbox);
146        std::fs::create_dir_all(&sandbox)
147            .with_context(|| format!("creating sandbox {}", sandbox.display()))?;
148
149        let cfg = agent::AgentConfig {
150            sandbox: sandbox.clone(),
151            max_turns: self.max_turns,
152            command_timeout: self.command_timeout,
153            request_timeout: self.request_timeout,
154            max_tokens: self.max_tokens,
155            cargo_target_dir: self.cargo_target_dir.clone(),
156        };
157
158        let started = Instant::now();
159        let transcript = agent::run_task(&handle, &cfg, PROMPT).await?;
160        // 2026-09-26: Taken before the scorer runs.
161        let agent_wall_s = started.elapsed().as_secs_f64();
162        handle.status(format!("run {index}: scoring"));
163        let web = score::webserver_test(
164            &sandbox,
165            self.cargo_target_dir.as_deref(),
166            self.build_timeout,
167            self.serve_timeout,
168        )
169        .await;
170        // 2026-09-26: Taken after the scorer, so `wall_s` includes it.
171        let wall_s = started.elapsed().as_secs_f64();
172        let directions = score::followed_directions(&transcript.commands, &sandbox);
173
174        let mut note = web.error.clone();
175        if transcript.hit_turn_cap {
176            note = format!("turn cap ({}) reached; {note}", self.max_turns);
177        }
178        // 2026-09-26: Name the directions the commands did not show, not only
179        // their count.
180        let missing = directions.missing();
181        if !missing.is_empty() {
182            note = format!("missing: {}; {note}", missing.join(", "));
183        }
184        Ok(IterationRow {
185            index,
186            wall_s,
187            agent_wall_s,
188            webserver_ok: web.webserver_ok,
189            directions,
190            turns: transcript.turns,
191            tool_calls: transcript.tool_calls,
192            completion_tokens: transcript.completion_tokens,
193            hit_turn_cap: transcript.hit_turn_cap,
194            truncated_turns: transcript.truncated_turns,
195            unparsed_call_turns: transcript.unparsed_call_turns,
196            note: one_line(note),
197        })
198    }
199
200    /// 2026-09-26: The record's metrics, computed from the same rows as the
201    /// summary and the verdict.
202    fn metrics(&self) -> std::collections::BTreeMap<String, f64> {
203        let n = self.rows.len();
204        let mut m = std::collections::BTreeMap::new();
205        m.insert("iterations".to_string(), n as f64);
206        m.insert(
207            "webserver_ok".to_string(),
208            self.rows.iter().filter(|r| r.webserver_ok).count() as f64,
209        );
210        m.insert(
211            "followed_directions".to_string(),
212            self.rows.iter().filter(|r| r.directions.overall()).count() as f64,
213        );
214        m.insert("sum_wall_s".to_string(), self.total_wall());
215        m.insert("sum_agent_wall_s".to_string(), self.total_agent_wall());
216        m.insert("sum_turns".to_string(), self.total_turns() as f64);
217        m.insert("sum_tool_calls".to_string(), self.total_tool_calls() as f64);
218        m.insert(
219            "sum_completion_tokens".to_string(),
220            self.total_completion_tokens() as f64,
221        );
222        // 2026-09-26: Absent, not 0.0, for a zero-turn tier: a 0.0 would read as
223        // the best speed ever recorded.
224        if let Some(spt) = self.seconds_per_turn() {
225            m.insert("s_per_turn".to_string(), spt);
226        }
227        // 2026-09-26: Completion tokens per agent-wall second. No BENCH.toml
228        // entry bounds it.
229        if self.total_agent_wall() > 0.0 {
230            m.insert(
231                "decode_tps".to_string(),
232                self.total_completion_tokens() as f64 / self.total_agent_wall(),
233            );
234        }
235
236        m.extend(score::per_step_tallies(
237            &self.rows.iter().map(|r| &r.directions).collect::<Vec<_>>(),
238        ));
239        m.extend(trajectory_diagnostics(&self.rows));
240        m
241    }
242
243    fn verdict(&self) -> Verdict {
244        verdict::verdict(
245            &self.rows,
246            self.total_wall(),
247            self.total_agent_wall(),
248            self.wall_budget_s,
249            self.s_per_turn_budget,
250        )
251    }
252}
253
254/// 2026-09-26: How the iterations' loops ended: counts of turn-cap endings,
255/// truncated turns and unparsed-call turns, and the most turns any iteration
256/// took. The sums are 0 over an empty tier; the maximum is absent, because "no
257/// iteration ran" and "an iteration took zero turns" are different facts.
258fn trajectory_diagnostics(rows: &[IterationRow]) -> std::collections::BTreeMap<String, f64> {
259    let mut m = std::collections::BTreeMap::new();
260    m.insert(
261        "sum_hit_turn_cap".to_string(),
262        rows.iter().filter(|r| r.hit_turn_cap).count() as f64,
263    );
264    m.insert(
265        "sum_truncated_turns".to_string(),
266        rows.iter().map(|r| r.truncated_turns).sum::<usize>() as f64,
267    );
268    m.insert(
269        "sum_unparsed_call_turns".to_string(),
270        rows.iter().map(|r| r.unparsed_call_turns).sum::<usize>() as f64,
271    );
272    if let Some(max) = rows.iter().map(|r| r.turns).max() {
273        m.insert("max_iter_turns".to_string(), max as f64);
274    }
275    m
276}
277
278impl Plugin for AgenticWebserver {
279    fn metadata(&self) -> &'static PluginMetadata {
280        &METADATA
281    }
282
283    fn load(&mut self, handle: PluginHandle) -> impl Future<Output = Result<()>> + Send {
284        self.started = Some(Instant::now());
285        let store = handle.artifacts().clone();
286        self.handle = Some(handle.clone());
287        async move {
288            // 2026-09-26: Fail at load if `cargo` is missing, since nothing could
289            // be scored.
290            crate::python::run(std::path::Path::new("cargo"), &["--version"], None)
291                .await
292                .context(
293                    "cargo is not on PATH — this benchmark builds the code the model writes",
294                )?;
295            let root = store.runs_dir(DESCRIPTOR.id)?.join("sandbox");
296            std::fs::create_dir_all(&root)?;
297            self.sandbox_root = Some(root);
298            // 2026-09-26: Build the dependencies into the shared target dir the
299            // agent and the scorer use (see [`warm`]).
300            self.cargo_target_dir = Some(warm::prepare(&handle).await?);
301            Ok(())
302        }
303    }
304}
305
306impl Benchmark for AgenticWebserver {
307    fn descriptor(&self) -> &'static BenchmarkDescriptor {
308        &DESCRIPTOR
309    }
310
311    fn parameters(&self) -> Vec<ParamSpec> {
312        params::parameters()
313    }
314
315    fn configure(&mut self, values: &ParamValues) -> Result<()> {
316        let specs = self.parameters();
317        values.validate_against(&specs)?;
318        self.iterations = values.usize("iterations")?;
319        self.wall_budget_s = values.float("wall_budget_s")?;
320        self.s_per_turn_budget = values.float("s_per_turn_budget")?;
321        self.max_turns = values.usize("max_turns")?;
322        self.command_timeout = Duration::from_secs(values.usize("command_timeout_s")? as u64);
323        self.build_timeout = Duration::from_secs(values.usize("build_timeout_s")? as u64);
324        self.serve_timeout = Duration::from_secs(values.usize("serve_timeout_s")? as u64);
325        self.max_tokens = values.usize("max_tokens")?;
326        self.request_timeout = Duration::from_secs(values.usize("request_timeout_s")? as u64);
327        self.cursor = 0;
328        self.rows.clear();
329        Ok(())
330    }
331
332    async fn next(&mut self) -> Result<BenchmarkResult> {
333        let handle = self.handle()?.clone();
334        handle.check_cancelled()?;
335        let total = self.iterations as u64;
336
337        if !self.probed {
338            self.probed = true;
339            http::probe(handle.target(), Duration::from_secs(10))
340                .await
341                .context("endpoint probe failed — check the target URL and port")?;
342            // 2026-09-26: The probe shows a server is listening, not that it
343            // decodes; `sanity_check` asks it 2+2 before the first iteration.
344            preflight::sanity_check(&handle, Duration::from_secs(60)).await?;
345            let root = self.sandbox_root.clone().context("no sandbox root")?;
346            return Ok(BenchmarkResult::running("probe", self.elapsed())
347                .with_progress(0, total)
348                .log_line(LogLine::info(format!(
349                    "{total} iteration(s) · sandbox {}",
350                    root.display()
351                )))
352                .log_line(LogLine::warn(
353                    "this benchmark executes model-authored shell inside the sandbox",
354                )));
355        }
356
357        if self.cursor >= self.iterations {
358            if self.rows.is_empty() {
359                bail!("no iterations ran");
360            }
361            return Ok(BenchmarkResult {
362                status: RunStatus::Completed,
363                ..BenchmarkResult::running("done", self.elapsed())
364            }
365            .with_progress(total, total)
366            .with_summary(self.summary())
367            .with_table(self.table())
368            .with_metrics(self.metrics())
369            .with_verdict(self.verdict()));
370        }
371
372        let index = self.cursor;
373        handle.status(format!("run {index}/{}", self.iterations));
374        let row = self.run_iteration(index).await?;
375        let line = LogLine::info(format!(
376            "run {index}: {} · {}/6 steps · {:.1}s · {} turns{}",
377            if row.webserver_ok {
378                "webserver_ok"
379            } else {
380                "FAILED"
381            },
382            row.directions.met(),
383            row.wall_s,
384            row.turns,
385            if row.note.is_empty() {
386                String::new()
387            } else {
388                format!(" · {}", row.note)
389            }
390        ));
391        self.rows.push(row);
392        self.cursor += 1;
393        handle.progress(self.cursor as u64, total);
394        Ok(
395            BenchmarkResult::running(format!("run {index}"), self.elapsed())
396                .with_progress(self.cursor as u64, total)
397                .with_summary(self.summary())
398                .with_table(self.table())
399                .log_line(line),
400        )
401    }
402
403    /// 2026-09-26: Sandboxes are kept: after a failed iteration the code the
404    /// model wrote is the evidence. Each is wiped when its index runs again.
405    async fn cleanup(&mut self) -> Result<()> {
406        Ok(())
407    }
408}
409
410#[cfg(test)]
411#[path = "agentic_diagnostics_tests.rs"]
412mod diagnostics_tests;
413#[cfg(test)]
414#[path = "agentic_tests.rs"]
415mod tests;
metrale_bench/benchmarks/agentic/mod.rs

metrale_bench/benchmarks/agentic/
mod.rs
