metrale_bench/gate/
scoring.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Scoring one record against one baseline, the pure half of the
4//! gate check: `compare` judges one metric against one bound, and
5//! `check_record` resolves the record's own (hardware, checkpoint) pair and
6//! scores that entry's pins and bounds.
7//!
8//! Owner: bench gate (verdict).
9//! Invariants:
10//! - Nothing here reads the filesystem, git or the environment.
11//! - `check_record` returns `None` only when the entry declares at least one
12//!   bound and every pin and bound is met.
13
14use super::record::{GateBaseline, GateRecord};
15
16/// 2026-09-26: One metric's comparison, or why it cannot be judged.
17pub enum Comparison {
18    Pass,
19    Fail(String),
20    Skip(String),
21}
22
23/// 2026-09-26: Compare one recorded metric against its bound, widened by
24/// `noise` (0 when absent) on the failing side. No bound at all is `Skip`.
25pub fn compare(name: &str, value: f64, bound: &super::record::Bound) -> Comparison {
26    let noise = bound.noise.unwrap_or(0.0);
27    match (bound.min, bound.max) {
28        (Some(min), None) if value + noise >= min => Comparison::Pass,
29        (Some(min), None) => Comparison::Fail(format!(
30            "{name} {value:.2} is below the floor {min:.2} (noise {noise:.2})"
31        )),
32        (None, Some(max)) if value - noise <= max => Comparison::Pass,
33        (None, Some(max)) => Comparison::Fail(format!(
34            "{name} {value:.2} is above the ceiling {max:.2} (noise {noise:.2})"
35        )),
36        // 2026-09-26: Both bounds are a range, and equal bounds an exact pin
37        // with its own failure message.
38        (Some(min), Some(max)) if value + noise >= min && value - noise <= max => Comparison::Pass,
39        (Some(min), Some(max)) if (min - max).abs() < f64::EPSILON => Comparison::Fail(format!(
40            "{name} is {value:.0}, but this gate is pinned to exactly {min:.0} — \
41             the run measured something other than what the baseline describes"
42        )),
43        (Some(min), Some(max)) => Comparison::Fail(format!(
44            "{name} {value:.2} is outside [{min:.2}, {max:.2}] (noise {noise:.2})"
45        )),
46        (None, None) => Comparison::Skip(format!("{name} has no bound")),
47    }
48}
49
50/// 2026-09-26: Check one record against its baseline. `None` means every pin
51/// and bound passed; `Some` carries the failures. A record whose (hardware,
52/// model) pair has no baseline entry fails.
53pub fn check_record(record: &GateRecord, baseline: &GateBaseline) -> Option<Vec<String>> {
54    // 2026-09-26: Scored only against the record's own (hardware, model)
55    // pair; a bound measured on another box or checkpoint does not apply.
56    let hardware = record.hardware.gate_key();
57    let entry = match baseline.resolve(&hardware, Some(&record.target_model)) {
58        Ok((_, entry)) => entry,
59        Err(e) => return Some(vec![format!("{e:#}")]),
60    };
61    // 2026-09-26: An entry with no bounds fails: the metric loop below would
62    // otherwise pass any record.
63    if entry.metrics.is_empty() {
64        return Some(vec![format!(
65            "the baseline entry for {} on {hardware} declares no thresholds — \
66             there is nothing here for this run to have passed",
67            record.target_model
68        )]);
69    }
70    let mut problems = Vec::new();
71    // 2026-09-26: Every baseline serve pin must be on the record at the
72    // pinned value, and the record may carry no unpinned override. BENCH.toml
73    // edits invalidate no record (`coverage::NON_COMPILED_KERNEL_FILES`), so
74    // a new pin must not let an older record pass.
75    for (k, want) in &entry.serve_overrides {
76        match record.serve_overrides.get(k) {
77            Some(got) if got == want => {}
78            Some(got) => problems.push(format!(
79                "serve override {k}={got} does not match the baseline pin {k}={want}"
80            )),
81            None => problems.push(format!(
82                "serve override {k}={want} is pinned on the baseline but missing from the record"
83            )),
84        }
85    }
86    for (k, got) in &record.serve_overrides {
87        if !entry.serve_overrides.contains_key(k) {
88            problems.push(format!(
89                "serve override {k}={got} is present on the record but not pinned by the baseline"
90            ));
91        }
92    }
93    // 2026-09-26: Every baseline param pin must be on the record at the pinned
94    // value, for the same reason. Each comma-separated item is trimmed before
95    // comparing, so `1, 4, 8, 16` matches the pin `1,4,8,16`.
96    let normalize = |s: &str| s.split(',').map(str::trim).collect::<Vec<_>>().join(",");
97    for (k, want) in &entry.param_overrides {
98        match record.params.get(k) {
99            Some(got) if normalize(got) == normalize(want) => {}
100            Some(got) => problems.push(format!(
101                "param {k}={got} does not match the baseline pin {k}={want} — the run \
102                 measured a different instrument than the one these thresholds describe"
103            )),
104            None => problems.push(format!(
105                "param {k}={want} is pinned on the baseline but missing from the record"
106            )),
107        }
108    }
109    for (name, bound) in &entry.metrics {
110        let Some(value) = record.metrics.get(name) else {
111            problems.push(format!("{name}: missing from the record"));
112            continue;
113        };
114        match compare(name, *value, bound) {
115            Comparison::Pass => {}
116            Comparison::Fail(reason) => problems.push(reason),
117            Comparison::Skip(reason) => problems.push(reason),
118        }
119    }
120    if problems.is_empty() {
121        None
122    } else {
123        Some(problems)
124    }
125}
metrale_bench/gate/scoring.rs

metrale_bench/gate/
scoring.rs
