metrale_bench/benchmarks/serve_matrix/
score.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The serve matrix's bars and tally, as pure functions of the round
4//! outcomes.
5//!
6//! Owner: bench, serve matrix.
7//! Invariants:
8//! - No I/O.
9//! - An identity, codegen or tool-call signal that is `NotRun` fails its bar,
10//!   and a planned round with no result fails as `no-result`.
11
12use super::plan::Plan;
13
14/// 2026-09-26: Coherence probes that must pass. `bars` caps it at the number
15/// run, so with fewer probes every one must pass.
16pub const COHERENCE_MIN_PASS: usize = 2;
17
18/// 2026-09-26: Fraction below a stored baseline tok/s that counts as a regression.
19pub const TPS_TOLERANCE: f64 = 0.10;
20
21/// 2026-09-26: One probe's answer. `NotApplicable` passes only the tool-call bar,
22/// where it means the server refused the tools request with a 4xx; on the
23/// identity and codegen bars it fails.
24#[derive(Clone, Debug, Default, PartialEq, Eq)]
25pub enum Signal {
26    Pass,
27    Fail(String),
28    NotApplicable(String),
29    /// 2026-09-26: The probe was not run. The default, so a half-built
30    /// `Signals` fails its bars.
31    #[default]
32    NotRun,
33}
34
35impl Signal {
36    pub fn is_fail(&self) -> bool {
37        matches!(self, Signal::Fail(_))
38    }
39
40    pub fn text(&self) -> &str {
41        match self {
42            Signal::Pass => "PASS",
43            Signal::Fail(_) => "FAIL",
44            Signal::NotApplicable(_) => "N/A",
45            Signal::NotRun => "—",
46        }
47    }
48
49    /// 2026-09-26: The detail behind a `Fail` or `NotApplicable`.
50    pub fn detail(&self) -> Option<&str> {
51        match self {
52            Signal::Fail(d) | Signal::NotApplicable(d) => Some(d),
53            _ => None,
54        }
55    }
56}
57
58/// 2026-09-26: What one booted round measured.
59#[derive(Clone, Debug, Default, PartialEq)]
60pub struct Signals {
61    /// 2026-09-26: Whether the endpoint serves this round's checkpoint; see
62    /// [`super::probes::Coherence::identity`].
63    pub identity: Signal,
64    pub coherence_pass: usize,
65    pub coherence_total: usize,
66    pub codegen: Signal,
67    pub tool_call: Signal,
68    /// 2026-09-26: Reported, not gated: `bars` does not read it.
69    pub long_ctx: Signal,
70    /// 2026-09-26: Decode tokens/sec from `probes::tps_probe`: `None` when the
71    /// client TPOT was undefined, `Some(0.0)` when the request failed.
72    pub tps: Option<f64>,
73}
74
75/// 2026-09-26: What became of a planned round.
76#[derive(Clone, Debug, PartialEq)]
77pub enum Outcome {
78    /// 2026-09-26: Came up and was probed.
79    Probed(Box<Signals>),
80    /// 2026-09-26: Planned and attempted, but `serve` failed or the endpoint
81    /// did not answer. Fails as `did-not-boot`, never a skip.
82    BootFailed(String),
83    /// 2026-09-26: Planned but not attempted: the benchmark had no handle, host
84    /// or options (`run_round`). Fails as `no-result`.
85    NotReached,
86}
87
88/// 2026-09-26: One planned round's result.
89#[derive(Clone, Debug, PartialEq)]
90pub struct RoundResult {
91    pub label: String,
92    pub outcome: Outcome,
93    /// 2026-09-26: Stored baseline tok/s for this label, if this box has one.
94    pub baseline_tps: Option<f64>,
95}
96
97impl RoundResult {
98    pub fn signals(&self) -> Option<&Signals> {
99        match &self.outcome {
100            Outcome::Probed(s) => Some(s),
101            _ => None,
102        }
103    }
104
105    /// 2026-09-26: The bars this round failed. Empty means verified.
106    pub fn bars(&self) -> Vec<String> {
107        let signals = match &self.outcome {
108            // 2026-09-26: Worded apart: a boot failure and an unattempted round
109            // need different fixes.
110            Outcome::BootFailed(why) => return vec![format!("did-not-boot ({why})")],
111            Outcome::NotReached => return vec!["no-result".into()],
112            Outcome::Probed(s) => s,
113        };
114        let mut fails = Vec::new();
115        // 2026-09-26: A `NotRun` signal fails its bar; `Signals::default()` is
116        // all `NotRun`.
117        for (name, signal, allows_not_applicable) in [
118            ("wrong-model", &signals.identity, false),
119            ("codegen", &signals.codegen, false),
120            ("tool_call", &signals.tool_call, true),
121        ] {
122            match signal {
123                Signal::Pass => {}
124                Signal::NotApplicable(_) if allows_not_applicable => {}
125                Signal::NotApplicable(_) => fails.push(format!("{name}(not-applicable)")),
126                Signal::Fail(_) => fails.push(name.to_string()),
127                Signal::NotRun => fails.push(format!("{name}(not-probed)")),
128            }
129        }
130        let coherence_bar = COHERENCE_MIN_PASS.min(signals.coherence_total);
131        if signals.coherence_total == 0
132            || signals.coherence_pass < coherence_bar
133            || signals.coherence_pass > signals.coherence_total
134        {
135            fails.push(format!(
136                "coherence({}/{})",
137                signals.coherence_pass, signals.coherence_total
138            ));
139        }
140        if let Some(tps) = signals.tps {
141            if !tps.is_finite() {
142                fails.push("tps(non-finite)".into());
143            } else if tps <= 0.0 {
144                fails.push("tps(0)".into());
145            } else if let Some(base) = self
146                .baseline_tps
147                .filter(|baseline| baseline.is_finite() && *baseline > 0.0)
148            {
149                let floor = base * (1.0 - TPS_TOLERANCE);
150                if tps < floor {
151                    fails.push(format!("tps({tps:.1}<{floor:.1})"));
152                }
153            }
154        }
155        fails
156    }
157
158    /// 2026-09-26: The note for a positive, finite tok/s that had no valid
159    /// baseline to compare with.
160    pub fn tps_note(&self) -> Option<&'static str> {
161        let tps = self.signals()?.tps?;
162        if tps.is_finite()
163            && tps > 0.0
164            && self
165                .baseline_tps
166                .filter(|baseline| baseline.is_finite() && *baseline > 0.0)
167                .is_none()
168        {
169            return Some("no baseline — liveness only");
170        }
171        None
172    }
173}
174
175/// 2026-09-26: The whole matrix, scored.
176#[derive(Clone, Debug, Default, PartialEq)]
177pub struct Tally {
178    pub verified: usize,
179    pub planned: usize,
180    pub skipped: usize,
181    pub excluded: usize,
182    /// 2026-09-26: `(label, bars)` for every round below bar.
183    pub failures: Vec<(String, Vec<String>)>,
184}
185
186impl Tally {
187    pub fn passed(&self) -> bool {
188        self.planned > 0 && self.failures.is_empty()
189    }
190}
191
192/// 2026-09-26: Score every planned round against its result. Walks the plan, not
193/// the results, so a planned round with no result fails as `no-result`.
194pub fn tally(plan: &Plan, results: &[RoundResult]) -> Tally {
195    let mut out = Tally {
196        planned: plan.planned_count(),
197        skipped: plan.skipped().count(),
198        excluded: plan.excluded_count(),
199        ..Tally::default()
200    };
201    for round in plan.planned() {
202        let label = round.label();
203        let bars = match results.iter().find(|r| r.label == label) {
204            Some(r) => r.bars(),
205            None => vec!["no-result".into()],
206        };
207        if bars.is_empty() {
208            out.verified += 1;
209        } else {
210            out.failures.push((label, bars));
211        }
212    }
213    out
214}
215
216/// 2026-09-26: The verdict sentence: coverage, skips (up to three named), the
217/// filter count, and each failing round's bars.
218pub fn verdict_text(tally: &Tally, plan: &Plan) -> String {
219    let coverage = format!(
220        "{}/{} planned checkpoints verified",
221        tally.verified, tally.planned
222    );
223    let mut extra = Vec::new();
224    if tally.skipped > 0 {
225        let names: Vec<String> = plan
226            .skipped()
227            .take(3)
228            .map(|(r, why)| format!("{} ({})", r.model, why.reason()))
229            .collect();
230        extra.push(format!(
231            "{} not runnable on this box: {}{}",
232            tally.skipped,
233            names.join(", "),
234            if tally.skipped > 3 { ", …" } else { "" }
235        ));
236    }
237    if tally.excluded > 0 {
238        extra.push(format!("{} outside the filter", tally.excluded));
239    }
240    let tail = if extra.is_empty() {
241        String::new()
242    } else {
243        format!(" · {}", extra.join(" · "))
244    };
245    if tally.failures.is_empty() {
246        return format!("{coverage}{tail}");
247    }
248    let detail: Vec<String> = tally
249        .failures
250        .iter()
251        .map(|(label, bars)| format!("{label}: {}", bars.join(", ")))
252        .collect();
253    format!(
254        "{coverage}{tail} — {} below bar: {}",
255        tally.failures.len(),
256        detail.join(" · ")
257    )
258}
259
260#[cfg(test)]
261#[path = "score_tests.rs"]
262mod tests;
metrale_bench/benchmarks/serve_matrix/score.rs

metrale_bench/benchmarks/serve_matrix/
score.rs
