metrale_bench/
result.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: What a benchmark emits on every `next()`: the frame the TUI
4//! renders.
5//!
6//! The types are presentation-shaped but style-free: a cell carries a
7//! semantic [`CellStyle`], never a color; the TUI's `theme::cell_style` maps
8//! it onto the palette. Frames are serializable, so a run's last frame is
9//! kept in its run record under the artifact store's `runs/` and re-rendered
10//! by the History pane.
11//!
12//! Owner: bench (results).
13//! Invariants:
14//! - [`RunStatus::is_terminal`] is false only for `Running`.
15//! - [`BenchmarkResult::failed`] puts its reason in both the verdict and the
16//!   log.
17
18use std::collections::BTreeMap;
19use std::time::Duration;
20
21use serde::{Deserialize, Serialize};
22
23/// 2026-09-26: Where a run is. The `run()` stream ends after the first
24/// terminal status.
25#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
26pub enum RunStatus {
27    /// 2026-09-26: More frames are coming; `next()` will be called again.
28    Running,
29    /// 2026-09-26: The benchmark finished its work. Terminal.
30    Completed,
31    /// 2026-09-26: The benchmark stopped early and the result is not
32    /// trustworthy. Terminal.
33    Failed,
34}
35
36impl RunStatus {
37    pub fn is_terminal(self) -> bool {
38        !matches!(self, RunStatus::Running)
39    }
40}
41
42/// 2026-09-26: Semantic emphasis. The TUI maps these onto its palette.
43#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
44pub enum CellStyle {
45    #[default]
46    Neutral,
47    Dim,
48    /// 2026-09-26: The value this row exists to show (brand cyan in the
49    /// TUI).
50    Accent,
51    Good,
52    Warn,
53    Bad,
54}
55
56#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
57pub enum Align {
58    #[default]
59    Left,
60    Right,
61}
62
63#[derive(Clone, Debug, Serialize, Deserialize)]
64pub struct Column {
65    pub title: String,
66    pub align: Align,
67    /// 2026-09-26: Preferred width in cells.
68    pub width: u16,
69}
70
71impl Column {
72    pub fn left(title: impl Into<String>, width: u16) -> Self {
73        Self {
74            title: title.into(),
75            align: Align::Left,
76            width,
77        }
78    }
79    /// 2026-09-26: Right-aligned, for numeric columns.
80    pub fn right(title: impl Into<String>, width: u16) -> Self {
81        Self {
82            title: title.into(),
83            align: Align::Right,
84            width,
85        }
86    }
87}
88
89#[derive(Clone, Debug, Default, Serialize, Deserialize)]
90pub struct Cell {
91    pub text: String,
92    pub style: CellStyle,
93}
94
95impl Cell {
96    pub fn new(text: impl Into<String>) -> Self {
97        Self {
98            text: text.into(),
99            style: CellStyle::Neutral,
100        }
101    }
102    pub fn styled(text: impl Into<String>, style: CellStyle) -> Self {
103        Self {
104            text: text.into(),
105            style,
106        }
107    }
108}
109
110#[derive(Clone, Debug, Serialize, Deserialize)]
111pub struct ResultTable {
112    pub title: String,
113    pub columns: Vec<Column>,
114    pub rows: Vec<Vec<Cell>>,
115}
116
117impl ResultTable {
118    pub fn new(title: impl Into<String>, columns: Vec<Column>) -> Self {
119        Self {
120            title: title.into(),
121            columns,
122            rows: Vec::new(),
123        }
124    }
125    pub fn push(&mut self, row: Vec<Cell>) {
126        self.rows.push(row);
127    }
128}
129
130/// 2026-09-26: A headline number for the tile row above the table.
131#[derive(Clone, Debug, Serialize, Deserialize)]
132pub struct Stat {
133    pub label: String,
134    pub value: String,
135    pub unit: String,
136    pub style: CellStyle,
137}
138
139impl Stat {
140    pub fn new(
141        label: impl Into<String>,
142        value: impl Into<String>,
143        unit: impl Into<String>,
144    ) -> Self {
145        Self {
146            label: label.into(),
147            value: value.into(),
148            unit: unit.into(),
149            style: CellStyle::Neutral,
150        }
151    }
152    pub fn with_style(mut self, style: CellStyle) -> Self {
153        self.style = style;
154        self
155    }
156}
157
158/// 2026-09-26: A gate outcome. `Info` is for benchmarks that measure without
159/// gating, so a sweep never renders a PASS it did not earn.
160#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
161pub enum VerdictKind {
162    Pass,
163    Fail,
164    Info,
165}
166
167#[derive(Clone, Debug, Serialize, Deserialize)]
168pub struct Verdict {
169    pub kind: VerdictKind,
170    /// 2026-09-26: Why. State the measured value against the threshold; a
171    /// bare "FAIL" sends the reader back to the raw logs.
172    pub reason: String,
173}
174
175impl Verdict {
176    pub fn pass(reason: impl Into<String>) -> Self {
177        Self {
178            kind: VerdictKind::Pass,
179            reason: reason.into(),
180        }
181    }
182    pub fn fail(reason: impl Into<String>) -> Self {
183        Self {
184            kind: VerdictKind::Fail,
185            reason: reason.into(),
186        }
187    }
188    pub fn info(reason: impl Into<String>) -> Self {
189        Self {
190            kind: VerdictKind::Info,
191            reason: reason.into(),
192        }
193    }
194}
195
196#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
197pub enum LogLevel {
198    Debug,
199    Info,
200    Warn,
201    Error,
202}
203
204#[derive(Clone, Debug, Serialize, Deserialize)]
205pub struct LogLine {
206    pub level: LogLevel,
207    pub text: String,
208}
209
210impl LogLine {
211    pub fn info(text: impl Into<String>) -> Self {
212        Self {
213            level: LogLevel::Info,
214            text: text.into(),
215        }
216    }
217    pub fn warn(text: impl Into<String>) -> Self {
218        Self {
219            level: LogLevel::Warn,
220            text: text.into(),
221        }
222    }
223    pub fn error(text: impl Into<String>) -> Self {
224        Self {
225            level: LogLevel::Error,
226            text: text.into(),
227        }
228    }
229}
230
231/// 2026-09-26: One frame of a benchmark run.
232#[derive(Clone, Debug, Serialize, Deserialize)]
233pub struct BenchmarkResult {
234    pub status: RunStatus,
235    /// 2026-09-26: What is happening right now, e.g. `"warmup"`.
236    pub phase: String,
237    /// 2026-09-26: `(done, total)` for the progress bar; `None` when the
238    /// total is unknown.
239    pub progress: Option<(u64, u64)>,
240    pub summary: Vec<Stat>,
241    pub table: Option<ResultTable>,
242    pub verdict: Option<Verdict>,
243    /// 2026-09-26: Raw headline numbers, keyed by stable metric name. The
244    /// gate record copies them and scores them against BENCH.toml bounds, so
245    /// the gate does not parse the formatted `summary` strings.
246    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
247    pub metrics: BTreeMap<String, f64>,
248    /// 2026-09-26: Lines appended to the run log since the previous frame
249    /// (not cumulative).
250    pub log: Vec<LogLine>,
251    pub elapsed: Duration,
252    /// 2026-09-26: What state the box was in before and after this run.
253    ///
254    /// The executor ([`crate::executor`]) stamps it on terminal frames, the
255    /// frame [`crate::RunRecord`] keeps and the gate record is built from.
256    /// `None` means unmeasured, which is not the same as a healthy box.
257    #[serde(default, skip_serializing_if = "Option::is_none")]
258    pub hardware_state: Option<crate::hardware::HardwareStateReport>,
259    /// 2026-09-26: Content identity of the dataset the run scored against,
260    /// when the benchmark sets one (the MLPerf agentic leg does). Carried
261    /// into the gate record: the metrics map is f64-only, so without this a
262    /// record could pin a draw's size but not its content.
263    #[serde(default, skip_serializing_if = "Option::is_none")]
264    pub dataset_fingerprint: Option<String>,
265}
266
267impl BenchmarkResult {
268    pub fn running(phase: impl Into<String>, elapsed: Duration) -> Self {
269        Self {
270            status: RunStatus::Running,
271            phase: phase.into(),
272            progress: None,
273            summary: Vec::new(),
274            table: None,
275            verdict: None,
276            metrics: BTreeMap::new(),
277            log: Vec::new(),
278            elapsed,
279            hardware_state: None,
280            dataset_fingerprint: None,
281        }
282    }
283
284    pub fn completed(phase: impl Into<String>, elapsed: Duration) -> Self {
285        Self {
286            status: RunStatus::Completed,
287            ..Self::running(phase, elapsed)
288        }
289    }
290
291    pub fn failed(phase: impl Into<String>, reason: impl Into<String>, elapsed: Duration) -> Self {
292        let reason = reason.into();
293        Self {
294            status: RunStatus::Failed,
295            verdict: Some(Verdict::fail(reason.clone())),
296            log: vec![LogLine::error(reason)],
297            ..Self::running(phase, elapsed)
298        }
299    }
300
301    pub fn with_progress(mut self, done: u64, total: u64) -> Self {
302        self.progress = Some((done, total));
303        self
304    }
305    pub fn with_summary(mut self, summary: Vec<Stat>) -> Self {
306        self.summary = summary;
307        self
308    }
309    pub fn with_metrics(mut self, metrics: BTreeMap<String, f64>) -> Self {
310        self.metrics = metrics;
311        self
312    }
313    pub fn with_table(mut self, table: ResultTable) -> Self {
314        self.table = Some(table);
315        self
316    }
317    pub fn with_verdict(mut self, verdict: Verdict) -> Self {
318        self.verdict = Some(verdict);
319        self
320    }
321    pub fn with_log(mut self, log: Vec<LogLine>) -> Self {
322        self.log = log;
323        self
324    }
325    pub fn log_line(mut self, line: LogLine) -> Self {
326        self.log.push(line);
327        self
328    }
329    pub fn with_hardware_state(mut self, report: crate::hardware::HardwareStateReport) -> Self {
330        self.hardware_state = Some(report);
331        self
332    }
333}
334
335#[cfg(test)]
336mod tests {
337    use super::*;
338
339    #[test]
340    fn only_running_is_non_terminal() {
341        assert!(!RunStatus::Running.is_terminal());
342        assert!(RunStatus::Completed.is_terminal());
343        assert!(RunStatus::Failed.is_terminal());
344    }
345
346    #[test]
347    fn failed_carries_the_reason_into_both_verdict_and_log() {
348        let r = BenchmarkResult::failed("scoring", "endpoint refused", Duration::from_secs(1));
349        assert_eq!(r.status, RunStatus::Failed);
350        assert_eq!(r.phase, "scoring");
351        assert_eq!(r.elapsed, Duration::from_secs(1));
352        let verdict = r.verdict.as_ref().expect("failure verdict");
353        assert_eq!(verdict.kind, VerdictKind::Fail);
354        assert_eq!(verdict.reason, "endpoint refused");
355        assert_eq!(r.log.len(), 1);
356        assert_eq!(r.log[0].level, LogLevel::Error);
357        assert_eq!(r.log[0].text, "endpoint refused");
358    }
359
360    #[test]
361    fn frames_round_trip_through_json_for_the_history_pane() {
362        let mut table = ResultTable::new("Samples", vec![Column::right("Rate", 8)]);
363        table.push(vec![Cell::styled("119.3", CellStyle::Good)]);
364        let mut r = BenchmarkResult::completed("done", Duration::from_millis(1500))
365            .with_progress(2, 3)
366            .with_summary(vec![
367                Stat::new("Throughput", "119.3", "tok/s").with_style(CellStyle::Accent),
368            ])
369            .with_table(table)
370            .with_metrics(BTreeMap::from([("median_tps".into(), 119.3)]))
371            .with_verdict(Verdict::pass("median +0.4% (limit 3%)"))
372            .with_log(vec![LogLine::warn("clock dipped")])
373            .with_hardware_state(crate::hardware::HardwareStateReport::opened(
374                crate::hardware::Sensitivity::Correctness,
375                crate::hardware::HardwareState::default(),
376                None,
377            ));
378        r.dataset_fingerprint = Some("file-sha256:abc".into());
379        let json = serde_json::to_string(&r).unwrap();
380        let back: BenchmarkResult = serde_json::from_str(&json).unwrap();
381        assert_eq!(back.status, RunStatus::Completed);
382        assert_eq!(back.phase, "done");
383        assert_eq!(back.progress, Some((2, 3)));
384        assert_eq!(back.elapsed, Duration::from_millis(1500));
385        assert_eq!(back.summary.len(), 1);
386        assert_eq!(back.summary[0].label, "Throughput");
387        assert_eq!(back.summary[0].value, "119.3");
388        assert_eq!(back.summary[0].unit, "tok/s");
389        assert_eq!(back.summary[0].style, CellStyle::Accent);
390        let table = back.table.as_ref().expect("table");
391        assert_eq!(table.title, "Samples");
392        assert_eq!(table.columns.len(), 1);
393        assert_eq!(table.columns[0].title, "Rate");
394        assert_eq!(table.columns[0].align, Align::Right);
395        assert_eq!(table.columns[0].width, 8);
396        assert_eq!(table.rows.len(), 1);
397        assert_eq!(table.rows[0][0].text, "119.3");
398        assert_eq!(table.rows[0][0].style, CellStyle::Good);
399        let verdict = back.verdict.as_ref().expect("verdict");
400        assert_eq!(verdict.kind, VerdictKind::Pass);
401        assert_eq!(verdict.reason, "median +0.4% (limit 3%)");
402        assert_eq!(back.metrics, BTreeMap::from([("median_tps".into(), 119.3)]));
403        assert_eq!(back.log.len(), 1);
404        assert_eq!(back.log[0].level, LogLevel::Warn);
405        assert_eq!(back.log[0].text, "clock dipped");
406        assert!(back.hardware_state.is_some());
407        assert_eq!(back.dataset_fingerprint.as_deref(), Some("file-sha256:abc"));
408    }
409}
metrale_bench/result.rs

metrale_bench/
result.rs
