metrale_bench/benchmarks/serve_matrix/
report.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Rendering the matrix: one table row per roster entry, one tile row
4//! of counts, the verdict, and the per-round log line.
5//!
6//! Owner: bench, serve matrix.
7//! Invariants:
8//! - The table has a row for every round of the plan, skipped and excluded
9//!   ones included.
10
11use super::plan::Plan;
12use super::score::{Outcome, RoundResult, Signal, Tally};
13use crate::result::{Cell, CellStyle, Column, Stat};
14use crate::result::{ResultTable, Verdict};
15
16fn style_of(signal: &Signal) -> CellStyle {
17    match signal {
18        Signal::Pass => CellStyle::Good,
19        Signal::Fail(_) => CellStyle::Bad,
20        Signal::NotApplicable(_) => CellStyle::Warn,
21        Signal::NotRun => CellStyle::Dim,
22    }
23}
24
25fn signal_cell(signal: &Signal) -> Cell {
26    Cell::styled(signal.text(), style_of(signal))
27}
28
29/// 2026-09-26: The full matrix, including the rounds that were skipped and why.
30pub fn table(plan: &Plan, results: &[RoundResult]) -> ResultTable {
31    let mut t = ResultTable::new(
32        "SERVE MATRIX",
33        vec![
34            Column::left("Model × quant", 40),
35            Column::left("Serving", 8),
36            Column::left("Coherence", 10),
37            Column::left("Codegen", 8),
38            Column::left("Tools", 6),
39            Column::left("Long ctx", 9),
40            Column::right("tok/s", 8),
41            Column::left("Verdict", 28),
42        ],
43    );
44    for round in &plan.rounds {
45        let label = round.label();
46        if let Some(why) = round.skipped {
47            t.push(skipped_row(&label, why.reason()));
48            continue;
49        }
50        if round.excluded {
51            t.push(skipped_row(&label, "outside the filter"));
52            continue;
53        }
54        let found = results.iter().find(|r| r.label == label);
55        match found {
56            Some(r) => t.push(result_row(&label, r)),
57            None => t.push(missing_result_row(&label)),
58        }
59    }
60    t
61}
62
63fn skipped_row(label: &str, why: &str) -> Vec<Cell> {
64    let mut row = vec![Cell::new(label)];
65    row.extend((0..6).map(|_| Cell::styled("—", CellStyle::Dim)));
66    row.push(Cell::styled(format!("SKIP · {why}"), CellStyle::Dim));
67    row
68}
69
70fn missing_result_row(label: &str) -> Vec<Cell> {
71    let mut row = vec![Cell::new(label)];
72    row.extend((0..6).map(|_| Cell::styled("—", CellStyle::Dim)));
73    row.push(Cell::styled(
74        "FAIL · no result — did not run",
75        CellStyle::Bad,
76    ));
77    row
78}
79
80fn result_row(label: &str, r: &RoundResult) -> Vec<Cell> {
81    let bars = r.bars();
82    let verdict = if bars.is_empty() {
83        Cell::styled("PASS", CellStyle::Good)
84    } else {
85        Cell::styled(format!("FAIL · {}", bars.join(", ")), CellStyle::Bad)
86    };
87    let Some(s) = r.signals() else {
88        let mut row = vec![Cell::new(label)];
89        row.extend((0..6).map(|_| Cell::styled("—", CellStyle::Dim)));
90        row.push(verdict);
91        return row;
92    };
93    let coherence = format!("{}/{}", s.coherence_pass, s.coherence_total);
94    let coherence_style = if s.coherence_pass == s.coherence_total {
95        CellStyle::Good
96    } else {
97        CellStyle::Bad
98    };
99    let tps = match s.tps {
100        Some(v) if v > 0.0 => Cell::styled(format!("{v:.1}"), CellStyle::Accent),
101        Some(_) => Cell::styled("0", CellStyle::Bad),
102        None => Cell::styled("—", CellStyle::Dim),
103    };
104    vec![
105        Cell::new(label),
106        // 2026-09-26: Whether the endpoint served this round's checkpoint.
107        signal_cell(&s.identity),
108        Cell::styled(coherence, coherence_style),
109        signal_cell(&s.codegen),
110        signal_cell(&s.tool_call),
111        signal_cell(&s.long_ctx),
112        tps,
113        verdict,
114    ]
115}
116
117pub fn summary(tally: &Tally) -> Vec<Stat> {
118    vec![
119        Stat::new(
120            "Verified",
121            format!("{}/{}", tally.verified, tally.planned),
122            "planned",
123        )
124        // 2026-09-26: 0/0 is "nothing was measured", which the verdict renders
125        // as Info, so the tile is dimmed rather than red.
126        .with_style(match (tally.planned, tally.passed()) {
127            (0, _) => CellStyle::Dim,
128            (_, true) => CellStyle::Good,
129            (_, false) => CellStyle::Bad,
130        }),
131        Stat::new("Below bar", tally.failures.len().to_string(), "").with_style(
132            if tally.failures.is_empty() {
133                CellStyle::Dim
134            } else {
135                CellStyle::Bad
136            },
137        ),
138        Stat::new("Not runnable", tally.skipped.to_string(), "on this box")
139            .with_style(CellStyle::Dim),
140        Stat::new("Filtered out", tally.excluded.to_string(), "").with_style(CellStyle::Dim),
141    ]
142}
143
144/// 2026-09-26: PASS/FAIL, or `Info` when no round was planned.
145pub fn verdict(tally: &Tally, plan: &Plan) -> Verdict {
146    let text = super::score::verdict_text(tally, plan);
147    if tally.planned == 0 {
148        return Verdict::info(format!(
149            "no checkpoint on this box is runnable — nothing was measured. {text}"
150        ));
151    }
152    if tally.passed() {
153        Verdict::pass(text)
154    } else {
155        Verdict::fail(text)
156    }
157}
158
159/// 2026-09-26: The per-round log line: enough to attribute a failure without the table.
160pub fn round_line(r: &RoundResult) -> String {
161    match &r.outcome {
162        Outcome::BootFailed(why) => format!("{}: DID NOT BOOT — {why}", r.label),
163        Outcome::NotReached => format!("{}: not reached", r.label),
164        Outcome::Probed(s) => {
165            let bars = r.bars();
166            let mut line = format!(
167                "{}: serving {} · coherence {}/{} · codegen {} · tools {} · long-ctx {} · {}",
168                r.label,
169                s.identity.text(),
170                s.coherence_pass,
171                s.coherence_total,
172                s.codegen.text(),
173                s.tool_call.text(),
174                s.long_ctx.text(),
175                match s.tps {
176                    Some(v) => format!("{v:.1} tok/s"),
177                    None => "tok/s unmeasured (one SSE delta)".into(),
178                }
179            );
180            if let Some(note) = r.tps_note() {
181                line.push_str(&format!(" ({note})"));
182            }
183            if !bars.is_empty() {
184                line.push_str(&format!(" → FAIL {}", bars.join(", ")));
185                // 2026-09-26: The detail of every failed signal, so the line
186                // stands alone.
187                let why: Vec<&str> = [&s.identity, &s.codegen, &s.tool_call, &s.long_ctx]
188                    .iter()
189                    .filter(|sig| sig.is_fail())
190                    .filter_map(|sig| sig.detail())
191                    .collect();
192                if !why.is_empty() {
193                    line.push_str(&format!(" — {}", why.join("; ")));
194                }
195            }
196            line
197        }
198    }
199}
200
201#[cfg(test)]
202#[path = "report_tests.rs"]
203mod tests;
metrale_bench/benchmarks/serve_matrix/report.rs

metrale_bench/benchmarks/serve_matrix/
report.rs
