metrale_bench/
benchmark.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The [`Benchmark`] trait, a [`Plugin`] driven one step at a time,
4//! and [`BenchmarkDescriptor`], its static identity.
5//!
6//! A benchmark owns its phase state and does one step of work per
7//! [`Benchmark::next`], returning a frame. It must not block the runtime or
8//! loop to completion inside one step: the driver checks for cancellation only
9//! between frames. [`Benchmark::run`] delegates to [`crate::executor::drive`],
10//! the same loop the executor uses.
11//!
12//! Owner: bench.
13//! Invariants: none beyond the types.
14
15use std::future::Future;
16
17use anyhow::Result;
18use futures::Stream;
19
20use crate::dynamic::DynBenchmark;
21use crate::hardware::Sensitivity;
22use crate::params::{ParamSpec, ParamValues};
23use crate::plugin::Plugin;
24use crate::result::BenchmarkResult;
25
26/// 2026-09-26: Static identity of a benchmark, and how to construct one. The
27/// registry (`registry::ALL`), the TUI list pane and the run directories
28/// (`ArtifactStore::runs_dir`) all read it.
29pub struct BenchmarkDescriptor {
30    /// 2026-09-26: Stable and filename-safe (`[a-z0-9-]`, checked by
31    /// `registry::tests::ids_are_unique_and_filename_safe`). Names the run
32    /// directory `<home>/runs/<id>/`.
33    pub id: &'static str,
34    pub name: &'static str,
35    /// 2026-09-26: One line for the suite list.
36    pub summary: &'static str,
37    /// 2026-09-26: A paragraph for the detail pane: what it measures and what
38    /// it costs.
39    pub detail: &'static str,
40    /// 2026-09-26: Rough wall time at default parameters, for display, e.g.
41    /// `"~15 min"`.
42    pub duration_hint: &'static str,
43    /// 2026-09-26: Expected wall time at default parameters, in seconds: the
44    /// certification planner's estimate when no completed run of this id has
45    /// been measured (`bench_certify::plan::units`). Zero only on descriptors
46    /// whose `duration_hint` starts with `unrunnable`
47    /// (`registry::tests::every_runnable_benchmark_declares_an_expected_duration`).
48    pub expected_secs: u64,
49    /// 2026-09-26: When this benchmark's measurement last changed (new
50    /// thresholds, prompt set or scoring rule), as `YYYY-MM-DD`; not when its
51    /// code was edited. It tells a reader whether two runs are comparable.
52    pub updated: &'static str,
53    /// 2026-09-26: True when starting has a side effect beyond load on the
54    /// endpoint. The TUI (`tui::bench_keys`) and `bench certify` (`--yes`)
55    /// require an explicit confirmation for these.
56    pub needs_confirmation: bool,
57    /// 2026-09-26: The checkpoint families this benchmark is defined on, if
58    /// any. The executor's endpoint check (`coherence::probe_for`) warns on a
59    /// mismatch and never refuses the run. `None` means the benchmark measures
60    /// whatever it is pointed at.
61    pub intended_for: Option<ModelExpectation>,
62    /// 2026-09-26: Parameters whose run-time value comes from the selected
63    /// model variant's committed bound, as `(param key, metric key)` pairs, for
64    /// benchmarks that compute their own verdict against a knob that is also a
65    /// `BENCH.toml` threshold (the agentic gate's `wall_budget_s` against
66    /// `sum_wall_s`). A gate run (`bench_resolve::apply_threshold_params`) and
67    /// the TUI derive the value from the variant's bound; an explicit `--param`
68    /// wins.
69    pub threshold_params: &'static [(&'static str, &'static str)],
70    /// 2026-09-26: Whether this benchmark's number is a speed number, and so
71    /// corruptible by the state of the box that produced it. A required field
72    /// of the descriptor, not a list in the hardware policy, so every new
73    /// benchmark has to answer it. See [`crate::hardware::policy`].
74    pub sensitivity: Sensitivity,
75    pub ctor: fn() -> Box<dyn DynBenchmark>,
76}
77
78/// 2026-09-26: Which checkpoints a benchmark's numbers mean something for.
79#[derive(Clone, Copy, Debug)]
80pub struct ModelExpectation {
81    /// 2026-09-26: Lower-case substrings identifying an acceptable checkpoint
82    /// family, not an exact id: [`ModelExpectation::accepts`] matches any model
83    /// id whose lower-cased form contains one of them, whatever its org prefix
84    /// or quantization suffix.
85    pub families: &'static [&'static str],
86    /// 2026-09-26: Appended to the mismatch warning: which model the benchmark
87    /// is defined on, and what running it elsewhere means.
88    pub note: &'static str,
89}
90
91impl ModelExpectation {
92    /// 2026-09-26: Does `model` belong to a family this benchmark is defined on?
93    pub fn accepts(&self, model: &str) -> bool {
94        let lowered = model.to_lowercase();
95        self.families.iter().any(|f| lowered.contains(f))
96    }
97}
98
99impl BenchmarkDescriptor {
100    pub fn build(&self) -> Box<dyn DynBenchmark> {
101        (self.ctor)()
102    }
103}
104
105pub trait Benchmark: Plugin {
106    fn descriptor(&self) -> &'static BenchmarkDescriptor;
107
108    /// 2026-09-26: The parameters shown before the run starts, so the user can
109    /// change them. Their defaults are in the returned specs.
110    fn parameters(&self) -> Vec<ParamSpec>;
111
112    /// 2026-09-26: Receive the edited values. Validate here and return an
113    /// error naming the offending field; the executor calls `next()` only after
114    /// this returns `Ok`.
115    fn configure(&mut self, values: &ParamValues) -> Result<()>;
116
117    /// 2026-09-26: Drive `next()` to completion, streaming every frame. The
118    /// stream ends after the first terminal [`crate::RunStatus`], or after an
119    /// error. It does not call `cleanup()`.
120    fn run(&mut self) -> impl Stream<Item = Result<BenchmarkResult>> + '_
121    where
122        Self: Sized + Send,
123    {
124        crate::executor::drive(self)
125    }
126
127    /// 2026-09-26: One step of work, returning the frame for that step.
128    fn next(&mut self) -> impl Future<Output = Result<BenchmarkResult>> + Send;
129
130    /// 2026-09-26: Release whatever the run acquired. The executor calls it
131    /// once on every return path of a run: completion, failure, cancellation,
132    /// a failed setup and a refused hardware precheck.
133    fn cleanup(&mut self) -> impl Future<Output = Result<()>> + Send {
134        async { Ok(()) }
135    }
136}
137
138#[cfg(test)]
139#[path = "benchmark_desc_tests.rs"]
140mod desc_tests;
metrale_bench/benchmark.rs

metrale_bench/
benchmark.rs
