metrale_bench/benchmarks/
baseline.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Per-benchmark baselines: the metrics of an earlier run, stored as
4//! JSON in the artifact store's `runs/<benchmark_id>/` directory, for the
5//! benchmarks that compare a run against their own history (`ttft`,
6//! `serve_matrix`). Each file records the endpoint and model it was measured
7//! against.
8//!
9//! Owner: bench.
10//! Invariants: `load_for` with a model named never returns a baseline whose
11//! `model` is a different one.
12
13use std::collections::BTreeMap;
14use std::time::{SystemTime, UNIX_EPOCH};
15
16use anyhow::{Context, Result};
17use serde::{Deserialize, Serialize};
18
19use crate::artifacts::ArtifactStore;
20
21#[derive(Clone, Debug, Serialize, Deserialize)]
22pub struct Baseline {
23    /// 2026-09-26: Unix seconds; `age_text` shows it.
24    pub recorded_at: u64,
25    /// 2026-09-26: The endpoint and model the numbers were measured against.
26    /// `ttft_verdict` reports a baseline from another box or model instead of
27    /// comparing against it.
28    pub target: String,
29    pub model: String,
30    pub metrics: BTreeMap<String, f64>,
31}
32
33impl Baseline {
34    pub fn get(&self, key: &str) -> Option<f64> {
35        self.metrics.get(key).copied()
36    }
37
38    /// 2026-09-26: Age as text: `just now`, `N min old`, `N h old` or `N d old`.
39    pub fn age_text(&self) -> String {
40        let now = now_secs();
41        let secs = now.saturating_sub(self.recorded_at);
42        match secs {
43            0..=90 => "just now".into(),
44            91..=5400 => format!("{} min old", secs / 60),
45            5401..=172_800 => format!("{} h old", secs / 3600),
46            _ => format!("{} d old", secs / 86_400),
47        }
48    }
49}
50
51fn now_secs() -> u64 {
52    SystemTime::now()
53        .duration_since(UNIX_EPOCH)
54        .map(|d| d.as_secs())
55        .unwrap_or(0)
56}
57
58/// 2026-09-26: Where one model's baseline for `benchmark_id` lives:
59/// `baseline-<variant_slug>.json`, or `baseline.json` when no model is named.
60/// Keyed by model because one gate can serve several checkpoints, and with one
61/// shared file each variant's run would replace the other's baseline.
62fn path(
63    store: &ArtifactStore,
64    benchmark_id: &str,
65    model: Option<&str>,
66) -> Result<std::path::PathBuf> {
67    let name = match model {
68        Some(m) => format!("baseline-{}.json", crate::gate::record::variant_slug(m)),
69        None => "baseline.json".to_string(),
70    };
71    Ok(store.runs_dir(benchmark_id)?.join(name))
72}
73
74/// 2026-09-26: The stored baseline, if any. An unreadable or corrupt file reads
75/// as absent: a run without a baseline still works, a refusal to start would
76/// not.
77pub fn load(store: &ArtifactStore, benchmark_id: &str) -> Option<Baseline> {
78    // 2026-09-26: `baseline.json` first, else the model-keyed file if exactly
79    // one exists. With two or more, none is returned rather than a guess.
80    if let Ok(p) = path(store, benchmark_id, None)
81        && let Ok(text) = std::fs::read_to_string(p)
82        && let Ok(b) = serde_json::from_str::<Baseline>(&text)
83    {
84        return Some(b);
85    }
86    let dir = store.runs_dir(benchmark_id).ok()?;
87    let mut candidate = None;
88    for e in std::fs::read_dir(dir).ok()? {
89        let path = e.ok()?.path();
90        let name = path.file_name()?.to_string_lossy().to_string();
91        if !(name.starts_with("baseline-") && name.ends_with(".json")) {
92            continue;
93        }
94        if candidate.is_some() {
95            return None;
96        }
97        candidate = Some(path);
98    }
99    let text = std::fs::read_to_string(candidate?).ok()?;
100    serde_json::from_str(&text).ok()
101}
102
103/// 2026-09-26: The baseline for one model: its model-keyed file, else
104/// `baseline.json` if that records the same model. With no model named,
105/// `baseline.json` alone.
106pub fn load_for(
107    store: &ArtifactStore,
108    benchmark_id: &str,
109    model: Option<&str>,
110) -> Option<Baseline> {
111    if let Some(m) = model
112        && let Ok(p) = path(store, benchmark_id, Some(m))
113        && let Ok(text) = std::fs::read_to_string(p)
114        && let Ok(b) = serde_json::from_str::<Baseline>(&text)
115        && b.model == m
116    {
117        return Some(b);
118    }
119    let p = path(store, benchmark_id, None).ok()?;
120    let text = std::fs::read_to_string(p).ok()?;
121    let b: Baseline = serde_json::from_str(&text).ok()?;
122    match model {
123        Some(m) if b.model != m => None,
124        _ => Some(b),
125    }
126}
127
128/// 2026-09-26: Writes the model-keyed baseline, replacing any earlier one. Call
129/// only after a run whose numbers should become the bar: a failed or partial
130/// leg stored here is compared against by every later run.
131pub fn save(
132    store: &ArtifactStore,
133    benchmark_id: &str,
134    target: &str,
135    model: &str,
136    metrics: BTreeMap<String, f64>,
137) -> Result<()> {
138    let baseline = Baseline {
139        recorded_at: now_secs(),
140        target: target.to_string(),
141        model: model.to_string(),
142        metrics,
143    };
144    let p = path(store, benchmark_id, Some(model))?;
145    std::fs::write(&p, serde_json::to_string_pretty(&baseline)?)
146        .with_context(|| format!("writing {}", p.display()))
147}
148
149#[cfg(test)]
150mod tests {
151    use super::*;
152
153    fn store(name: &str) -> ArtifactStore {
154        let d =
155            std::env::temp_dir().join(format!("metrale-baseline-{name}-{}", std::process::id()));
156        let _ = std::fs::remove_dir_all(&d);
157        ArtifactStore::with_root(d)
158    }
159
160    /// 2026-09-26: Two checkpoints of one gate keep separate baselines.
161    #[test]
162    fn two_models_of_one_gate_keep_separate_baselines() {
163        let s = store("variants");
164        let mut a = BTreeMap::new();
165        a.insert("median_ms".into(), 1677.9);
166        save(&s, "ttft-cold", "http://h:1", "Qwen/Qwen3.6-35B-A3B-FP8", a).unwrap();
167        let mut b = BTreeMap::new();
168        b.insert("median_ms".into(), 442.3);
169        save(
170            &s,
171            "ttft-cold",
172            "http://h:1",
173            "nvidia/Qwen3.6-35B-A3B-NVFP4",
174            b,
175        )
176        .unwrap();
177        let fp8 = load_for(&s, "ttft-cold", Some("Qwen/Qwen3.6-35B-A3B-FP8")).unwrap();
178        assert_eq!(fp8.get("median_ms"), Some(1677.9));
179        let nv = load_for(&s, "ttft-cold", Some("nvidia/Qwen3.6-35B-A3B-NVFP4")).unwrap();
180        assert_eq!(nv.get("median_ms"), Some(442.3));
181    }
182
183    /// 2026-09-26: A model with no baseline of its own reads as absent rather
184    /// than inheriting `baseline.json` from a different checkpoint.
185    #[test]
186    fn legacy_shared_baseline_is_not_inherited_by_another_model() {
187        let s = store("legacy");
188        let p = s.runs_dir("ttft-warm").unwrap().join("baseline.json");
189        let legacy = Baseline {
190            recorded_at: now_secs(),
191            target: "http://h:1".into(),
192            model: "Qwen/Qwen3.6-35B-A3B-FP8".into(),
193            metrics: BTreeMap::from([("median_ms".to_string(), 1600.0)]),
194        };
195        std::fs::write(&p, serde_json::to_string(&legacy).unwrap()).unwrap();
196        assert!(load_for(&s, "ttft-warm", Some("Qwen/Qwen3.6-35B-A3B-FP8")).is_some());
197        assert!(load_for(&s, "ttft-warm", Some("nvidia/Qwen3.6-35B-A3B-NVFP4")).is_none());
198    }
199
200    #[test]
201    fn round_trips_and_reports_the_target_it_came_from() {
202        let s = store("rt");
203        assert!(load(&s, "ttft-warm").is_none());
204        let mut m = BTreeMap::new();
205        m.insert("median_ms".into(), 812.5);
206        save(&s, "ttft-warm", "http://127.0.0.1:8888", "qwen", m).unwrap();
207        let b = load(&s, "ttft-warm").unwrap();
208        assert_eq!(b.get("median_ms"), Some(812.5));
209        assert_eq!(b.target, "http://127.0.0.1:8888");
210        assert_eq!(b.model, "qwen");
211        assert_eq!(b.age_text(), "just now");
212    }
213
214    #[test]
215    fn a_model_keyed_file_must_belong_to_the_model_in_its_name() {
216        let s = store("misfiled");
217        let p = path(&s, "ttft-warm", Some("wanted-model")).unwrap();
218        let wrong = Baseline {
219            recorded_at: now_secs(),
220            target: "http://h:1".into(),
221            model: "another-model".into(),
222            metrics: BTreeMap::from([("median_ms".to_string(), 1.0)]),
223        };
224        std::fs::write(p, serde_json::to_string(&wrong).unwrap()).unwrap();
225        assert!(load_for(&s, "ttft-warm", Some("wanted-model")).is_none());
226    }
227
228    #[test]
229    fn an_unkeyed_load_never_guesses_among_multiple_model_files() {
230        let s = store("ambiguous");
231        let corrupt = path(&s, "ttft-warm", Some("created-first")).unwrap();
232        std::fs::write(corrupt, "{ not json").unwrap();
233        save(
234            &s,
235            "ttft-warm",
236            "http://h:1",
237            "created-second",
238            BTreeMap::from([("median_ms".to_string(), 1.0)]),
239        )
240        .unwrap();
241        assert!(
242            load(&s, "ttft-warm").is_none(),
243            "two model-keyed files are ambiguous even when only one parses"
244        );
245    }
246
247    #[test]
248    fn a_corrupt_baseline_reads_as_absent() {
249        let s = store("corrupt");
250        let p = s.runs_dir("x").unwrap().join("baseline.json");
251        std::fs::write(p, "{ not json").unwrap();
252        assert!(load(&s, "x").is_none());
253    }
254}
metrale_bench/benchmarks/baseline.rs

metrale_bench/benchmarks/
baseline.rs
