metrale_bench/
history.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Run history: [`RunRecord`]s written by [`save`] and read by
4//! [`load`], [`load_all`] and [`find`], under the artifact store's
5//! `runs/<benchmark>` directory. A record built by [`RunRecord::new`] carries
6//! every parameter, the target, the source and the engine version beside the
7//! terminal frame. The headless driver and the TUI both save through
8//! [`save`].
9//!
10//! Owner: bench.
11//! Invariants:
12//! - [`save`] never overwrites another run: it claims a fresh
13//!   `run-<nanos>.json` with `create_new`, and the finished record replaces
14//!   only its own claim, by rename.
15//! - The readers skip files they cannot read or parse.
16
17use std::collections::BTreeMap;
18use std::path::{Path, PathBuf};
19use std::time::{SystemTime, UNIX_EPOCH};
20
21use anyhow::{Context, Result};
22use serde::{Deserialize, Serialize};
23
24use crate::artifacts::ArtifactStore;
25use crate::benchmark::BenchmarkDescriptor;
26use crate::params::{ParamSpec, ParamValues};
27use crate::plugin::TargetEndpoint;
28use crate::result::{BenchmarkResult, VerdictKind};
29
30/// 2026-09-26: Current record schema. `0` marks a bare-frame file read
31/// through `from_legacy`, or a record with no `schema` field.
32pub const SCHEMA: u32 = 1;
33
34/// 2026-09-26: Where a run was started from.
35#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
36#[serde(rename_all = "lowercase")]
37pub enum RunSource {
38    Tui,
39    Cli,
40    /// 2026-09-26: A bare-frame file, which carries no provenance, or a
41    /// record with no `source` field.
42    #[default]
43    Unknown,
44}
45
46/// 2026-09-26: One finished run, as stored. Only `benchmark_id`,
47/// `recorded_at` and `frame` are required and every other field defaults, so
48/// a bare [`BenchmarkResult`] (no `frame` key) never parses as a record.
49#[derive(Clone, Debug, Serialize, Deserialize)]
50pub struct RunRecord {
51    #[serde(default)]
52    pub schema: u32,
53    /// 2026-09-26: The file stem [`save`] claimed; [`find`] looks a run up by
54    /// it.
55    #[serde(default)]
56    pub run_id: String,
57    pub benchmark_id: String,
58    #[serde(default)]
59    pub benchmark_name: String,
60    /// 2026-09-26: Unix seconds when [`RunRecord::new`] assembled the record,
61    /// after the run ended. A bare-frame file takes the number in its stem.
62    pub recorded_at: u64,
63    /// 2026-09-26: Stored flat rather than as a `TargetEndpoint`, whose
64    /// constructor trims trailing slashes; [`RunRecord::target`] rebuilds it
65    /// through that constructor.
66    #[serde(default)]
67    pub target_url: String,
68    #[serde(default)]
69    pub target_model: String,
70    /// 2026-09-26: Every parameter, defaults included.
71    #[serde(default)]
72    pub params: BTreeMap<String, String>,
73    /// 2026-09-26: The serve overrides the caller put on the target. The CLI
74    /// sets them only under `--pull-request-gate`, to the served plan's merged
75    /// set (the baseline's `[benchmarks.serve_overrides]` and
76    /// `--serve-override`, after `--hermetic` expansion); the TUI sets none.
77    /// Empty therefore does not mean the endpoint had no overrides.
78    #[serde(default)]
79    pub serve_overrides: BTreeMap<String, String>,
80    #[serde(default)]
81    pub source: RunSource,
82    #[serde(default)]
83    pub metrale_version: String,
84    /// 2026-09-26: The run's terminal frame.
85    pub frame: BenchmarkResult,
86}
87
88impl RunRecord {
89    /// 2026-09-26: Assemble a record when the run ends. `run_id` stays empty
90    /// until [`save`] sets it.
91    pub fn new(
92        descriptor: &BenchmarkDescriptor,
93        values: &ParamValues,
94        target: &TargetEndpoint,
95        serve_overrides: BTreeMap<String, String>,
96        source: RunSource,
97        metrale_version: &str,
98        frame: BenchmarkResult,
99    ) -> Self {
100        Self {
101            schema: SCHEMA,
102            run_id: String::new(),
103            benchmark_id: descriptor.id.to_string(),
104            benchmark_name: descriptor.name.to_string(),
105            recorded_at: now_secs(),
106            target_url: target.base_url.clone(),
107            target_model: target.model.clone(),
108            params: values.to_strings(),
109            serve_overrides,
110            source,
111            metrale_version: metrale_version.to_string(),
112            frame,
113        }
114    }
115
116    /// 2026-09-26: The endpoint, rebuilt through `TargetEndpoint::new`.
117    pub fn target(&self) -> TargetEndpoint {
118        TargetEndpoint::new(&self.target_url, &self.target_model)
119    }
120
121    /// 2026-09-26: Rehydrate the stored parameters against a live schema
122    /// through `ParamValues::from_overrides`, which errors on an unknown key or
123    /// a value the spec's `ParamKind` rejects.
124    pub fn values(&self, specs: &[ParamSpec]) -> Result<ParamValues> {
125        let pairs = self
126            .params
127            .iter()
128            .map(|(k, v)| (k.as_str(), v.as_str()))
129            .collect::<Vec<_>>();
130        ParamValues::from_overrides(specs, pairs)
131    }
132
133    pub fn verdict_kind(&self) -> Option<VerdictKind> {
134        self.frame.verdict.as_ref().map(|v| v.kind)
135    }
136
137    /// 2026-09-26: True when `schema` is 0.
138    pub fn is_legacy(&self) -> bool {
139        self.schema == 0
140    }
141
142    /// 2026-09-26: Compact age, e.g. `3m ago`.
143    pub fn age_text(&self) -> String {
144        let secs = now_secs().saturating_sub(self.recorded_at);
145        match secs {
146            0..=59 => format!("{secs}s ago"),
147            60..=3599 => format!("{}m ago", secs / 60),
148            3600..=86_399 => format!("{}h ago", secs / 3600),
149            _ => format!("{}d ago", secs / 86_400),
150        }
151    }
152
153    fn from_legacy(benchmark_id: &str, run_id: &str, frame: BenchmarkResult) -> Self {
154        Self {
155            schema: 0,
156            recorded_at: run_id
157                .trim_start_matches("run-")
158                .parse()
159                .unwrap_or_default(),
160            run_id: run_id.to_string(),
161            benchmark_id: benchmark_id.to_string(),
162            benchmark_name: String::new(),
163            target_url: String::new(),
164            target_model: String::new(),
165            params: BTreeMap::new(),
166            serve_overrides: BTreeMap::new(),
167            source: RunSource::Unknown,
168            metrale_version: String::new(),
169            frame,
170        }
171    }
172}
173
174fn now_secs() -> u64 {
175    SystemTime::now()
176        .duration_since(UNIX_EPOCH)
177        .map(|d| d.as_secs())
178        .unwrap_or_default()
179}
180
181/// 2026-09-26: Write `record`, setting its `run_id` to the file stem chosen,
182/// and return the path.
183///
184/// The name is `run-<unix_nanos>`, zero-padded to 19 digits, so a filename
185/// sort is a time sort while the nanos have 19 digits (until 2286). A taken
186/// name moves the nanos up by one until a name is free.
187pub fn save(store: &ArtifactStore, record: &mut RunRecord) -> Result<PathBuf> {
188    let nanos = SystemTime::now()
189        .duration_since(UNIX_EPOCH)
190        .map(|d| d.as_nanos() as u64)
191        .unwrap_or_default();
192    save_at(store, record, nanos, |_| {})
193}
194
195/// 2026-09-26: [`save`] with the clock and a post-claim hook supplied, so
196/// tests can force a collision and inspect the claimed placeholder.
197fn save_at<F>(
198    store: &ArtifactStore,
199    record: &mut RunRecord,
200    mut nanos: u64,
201    after_claim: F,
202) -> Result<PathBuf>
203where
204    F: FnOnce(&Path),
205{
206    let dir = store.runs_dir(&record.benchmark_id)?;
207    // 2026-09-26: `create_new` claims the name atomically, so two processes
208    // saving into this directory cannot both take it.
209    let (run_id, path) = loop {
210        let id = format!("run-{nanos:019}");
211        let path = dir.join(format!("{id}.json"));
212        match std::fs::OpenOptions::new()
213            .write(true)
214            .create_new(true)
215            .open(&path)
216        {
217            Ok(_) => break (id, path),
218            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
219                nanos = nanos.saturating_add(1);
220            }
221            Err(e) => {
222                return Err(e).with_context(|| format!("claiming {}", path.display()));
223            }
224        }
225    };
226    record.run_id = run_id;
227    after_claim(&path);
228    let json = serde_json::to_string_pretty(&record).context("serializing the run record")?;
229    // 2026-09-26: Write beside the claim and rename over it, so a reader sees
230    // the empty placeholder or the whole record, never part of one.
231    let tmp = path.with_extension("json.tmp");
232    std::fs::write(&tmp, json).with_context(|| format!("writing {}", tmp.display()))?;
233    std::fs::rename(&tmp, &path).with_context(|| format!("publishing {}", path.display()))?;
234    Ok(path)
235}
236
237/// 2026-09-26: Every run for one benchmark, newest first. A file that cannot
238/// be read or parsed is skipped; a directory that cannot be created or read
239/// gives an empty list.
240pub fn load(store: &ArtifactStore, benchmark_id: &str) -> Vec<RunRecord> {
241    let Ok(dir) = store.runs_dir(benchmark_id) else {
242        return Vec::new();
243    };
244    let Ok(entries) = std::fs::read_dir(&dir) else {
245        return Vec::new();
246    };
247    let mut out: Vec<RunRecord> = entries
248        .flatten()
249        .filter_map(|e| read_one(benchmark_id, &e.path()))
250        .collect();
251    sort_newest_first(&mut out);
252    out
253}
254
255/// 2026-09-26: Every run for every registered benchmark, newest first across
256/// all of them.
257pub fn load_all(store: &ArtifactStore) -> Vec<RunRecord> {
258    let mut out: Vec<RunRecord> = crate::registry::all()
259        .iter()
260        .flat_map(|d| load(store, d.id))
261        .collect();
262    sort_newest_first(&mut out);
263    out
264}
265
266/// 2026-09-26: One run by its `run_id`, across every registered benchmark.
267pub fn find(store: &ArtifactStore, run_id: &str) -> Option<RunRecord> {
268    load_all(store).into_iter().find(|r| r.run_id == run_id)
269}
270
271fn sort_newest_first(records: &mut [RunRecord]) {
272    records.sort_by(|a, b| {
273        b.recorded_at
274            .cmp(&a.recorded_at)
275            .then_with(|| b.run_id.cmp(&a.run_id))
276    });
277}
278
279/// 2026-09-26: Parse one `run-*.json` file as a record, or else as a bare
280/// frame. Other files in the directory are ignored.
281fn read_one(benchmark_id: &str, path: &Path) -> Option<RunRecord> {
282    let stem = path.file_stem()?.to_str()?;
283    if !stem.starts_with("run-") || path.extension()? != "json" {
284        return None; // 2026-09-26: baseline files and the agentic sandbox live here too
285    }
286    let text = std::fs::read_to_string(path).ok()?;
287    if let Ok(mut record) = serde_json::from_str::<RunRecord>(&text) {
288        if record.run_id.is_empty() {
289            record.run_id = stem.to_string();
290        }
291        return Some(record);
292    }
293    let frame = serde_json::from_str::<BenchmarkResult>(&text).ok()?;
294    Some(RunRecord::from_legacy(benchmark_id, stem, frame))
295}
296
297#[cfg(test)]
298#[path = "history_tests.rs"]
299mod tests;
metrale_bench/history.rs

metrale_bench/
history.rs
