metrale_bench/benchmarks/agentic/
warm.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The shared warm cargo target dir, a port of the harness's
4//! `warm_cargo_cache.sh`. With a cold dir every generated Axum project compiles
5//! its whole dependency tree inside the scorer's build timeout. This builds a
6//! template project in the debug profile (for the agent's own `cargo test` and
7//! `cargo run`) and the release profile (for the scorer's `cargo build
8//! --release`), and returns the dir that the agent shell and the scorer both
9//! set as `CARGO_TARGET_DIR`.
10//!
11//! Owner: bench, agentic.
12//! Invariants: `prepare` returns the dir only after both profile builds
13//! succeeded; any failure is an error, never a fall back to a cold dir.
14
15use std::ffi::OsString;
16use std::path::{Path, PathBuf};
17use std::process::Stdio;
18use std::time::Duration;
19
20use anyhow::{Context, Result, bail};
21
22use crate::plugin::PluginHandle;
23
24/// 2026-09-26: Per `cargo` invocation; a guard so a wedged `cargo` cannot hang
25/// the benchmark.
26const PREWARM_TIMEOUT: Duration = Duration::from_secs(1800);
27
28/// 2026-09-26: Parses to the same TOML as the `Cargo.toml` heredoc in
29/// `warm_cargo_cache.sh` (asserted in the tests).
30pub const TEMPLATE_MANIFEST: &str = r#"[package]
31name = "metrale-warm-template"
32version = "0.1.0"
33edition = "2021"
34
35[dependencies]
36axum = { version = "0.8", features = ["json", "macros", "ws", "multipart"] }
37tokio = { version = "1", features = ["full"] }
38serde = { version = "1", features = ["derive"] }
39serde_json = "1"
40tower = { version = "0.5", features = ["full"] }
41tower-http = { version = "0.6", features = ["full"] }
42hyper = { version = "1", features = ["full"] }
43reqwest = { version = "0.12", features = ["json"] }
44anyhow = "1"
45thiserror = "2"
46tracing = "0.1"
47tracing-subscriber = { version = "0.3", features = ["env-filter"] }
48
49[dev-dependencies]
50reqwest = { version = "0.12", features = ["json"] }
51"#;
52
53/// 2026-09-26: The template's `main.rs`, a ping/pong server on
54/// `METRALE_HARNESS_PORT`; the same text as the `main.rs` heredoc in
55/// `warm_cargo_cache.sh` after its first comment line (asserted in the tests).
56pub const TEMPLATE_MAIN: &str = r#"use axum::{routing::get, Router};
57
58async fn ping() -> &'static str {
59    "pong"
60}
61
62#[tokio::main]
63async fn main() {
64    let _ = serde_json::json!({"ok": true});
65    let _v: tower::ServiceBuilder<tower::layer::util::Identity> = tower::ServiceBuilder::new();
66    let app = Router::new().route("/ping", get(ping));
67    let port: u16 = std::env::var("METRALE_HARNESS_PORT")
68        .unwrap_or_else(|_| "3001".to_string())
69        .parse()
70        .unwrap();
71    let listener = tokio::net::TcpListener::bind(("0.0.0.0", port)).await.unwrap();
72    axum::serve(listener, app).await.unwrap();
73}
74"#;
75
76/// 2026-09-26: Resolves `${VAR:-${HOME}/.cargo/<leaf>}` as the harness does: an
77/// empty `explicit` counts as unset, and a missing or empty `home` is an error.
78/// The callers read the environment.
79fn dir_from(explicit: Option<OsString>, home: Option<OsString>, leaf: &str) -> Result<PathBuf> {
80    if let Some(p) = explicit.filter(|p| !p.is_empty()) {
81        return Ok(PathBuf::from(p));
82    }
83    let home = home
84        .filter(|h| !h.is_empty())
85        .context("HOME is not set, so the shared warm cargo dir has no home")?;
86    Ok(PathBuf::from(home).join(".cargo").join(leaf))
87}
88
89/// 2026-09-26: `METRALE_WARM_TARGET_DIR`, else
90/// `$HOME/.cargo/metrale-warm-target`: the variable and default path of
91/// `warm_cargo_cache.sh`, `run_tier.sh` and `score_run.py`'s
92/// `_warm_target_dir` (asserted in the tests).
93pub fn warm_target_dir() -> Result<PathBuf> {
94    dir_from(
95        std::env::var_os("METRALE_WARM_TARGET_DIR"),
96        std::env::var_os("HOME"),
97        "metrale-warm-target",
98    )
99}
100
101/// 2026-09-26: `METRALE_WARM_TEMPLATE_DIR`, else
102/// `$HOME/.cargo/metrale-warm-template`, as in `warm_cargo_cache.sh`.
103pub fn template_dir() -> Result<PathBuf> {
104    dir_from(
105        std::env::var_os("METRALE_WARM_TEMPLATE_DIR"),
106        std::env::var_os("HOME"),
107        "metrale-warm-template",
108    )
109}
110
111/// 2026-09-26: Writes the template project. A file whose content already
112/// matches is left untouched.
113pub fn write_template(dir: &Path) -> Result<()> {
114    let src = dir.join("src");
115    std::fs::create_dir_all(&src)
116        .with_context(|| format!("creating warm template dir {}", src.display()))?;
117    write_if_changed(&dir.join("Cargo.toml"), TEMPLATE_MANIFEST)?;
118    write_if_changed(&src.join("main.rs"), TEMPLATE_MAIN)?;
119    Ok(())
120}
121
122/// 2026-09-26: Rewriting an identical file would bump its mtime and make cargo
123/// rebuild the template crate on every run.
124fn write_if_changed(path: &Path, content: &str) -> Result<()> {
125    if std::fs::read_to_string(path).is_ok_and(|old| old == content) {
126        return Ok(());
127    }
128    std::fs::write(path, content).with_context(|| format!("writing {}", path.display()))
129}
130
131/// 2026-09-26: One `cargo` invocation against the template, output into the
132/// warm dir. A non-zero exit, a failed start or `PREWARM_TIMEOUT` is an error.
133async fn cargo(args: &[&str], warm: &Path, manifest: &Path) -> Result<()> {
134    let mut cmd = tokio::process::Command::new("cargo");
135    cmd.args(args)
136        .arg("--manifest-path")
137        .arg(manifest)
138        .env("CARGO_TARGET_DIR", warm)
139        .stdin(Stdio::null())
140        .stdout(Stdio::null())
141        .stderr(Stdio::piped())
142        .kill_on_drop(true);
143    let out = tokio::time::timeout(PREWARM_TIMEOUT, cmd.output())
144        .await
145        .with_context(|| {
146            format!(
147                "`cargo {}` exceeded {}s while warming {}",
148                args.join(" "),
149                PREWARM_TIMEOUT.as_secs(),
150                warm.display()
151            )
152        })?
153        .with_context(|| format!("`cargo {}` could not start", args.join(" ")))?;
154    if !out.status.success() {
155        let err = String::from_utf8_lossy(&out.stderr);
156        let tail = err.lines().rev().take(8).collect::<Vec<_>>().join(" ");
157        bail!(
158            "`cargo {}` failed while warming {}: {}",
159            args.join(" "),
160            warm.display(),
161            super::super::one_line(tail)
162        );
163    }
164    Ok(())
165}
166
167/// 2026-09-26: Prepares the shared warm target dir and returns it. A failure is
168/// an error rather than a fall back to a cold dir, which would show only as
169/// slow or timed-out builds.
170pub async fn prepare(handle: &PluginHandle) -> Result<PathBuf> {
171    let warm = warm_target_dir()?;
172    let template = template_dir()?;
173    std::fs::create_dir_all(&warm)
174        .with_context(|| format!("creating warm target dir {}", warm.display()))?;
175    write_template(&template)?;
176    let manifest = template.join("Cargo.toml");
177
178    // 2026-09-26: `cargo test --no-run` builds the debug dependencies the
179    // agent's own `cargo test` and `cargo run` use; `cargo build --release`
180    // builds the release ones the scorer uses.
181    handle.check_cancelled()?;
182    handle.status(format!("warming cargo debug profile in {}", warm.display()));
183    cargo(&["test", "--no-run"], &warm, &manifest).await?;
184    handle.check_cancelled()?;
185    handle.status(format!(
186        "warming cargo release profile in {}",
187        warm.display()
188    ));
189    cargo(&["build", "--release"], &warm, &manifest).await?;
190    Ok(warm)
191}
192
193#[cfg(test)]
194#[path = "warm_tests.rs"]
195mod tests;
metrale_bench/benchmarks/agentic/warm.rs

metrale_bench/benchmarks/agentic/
warm.rs
