metrale_bench/
registry.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The benchmark suite, in the order the Benchmarks pane lists
4//! it, and lookup by id.
5//!
6//! Owner: bench (registry).
7//! Invariants: none beyond the types; the tests below check that ids are
8//! unique and filename-safe and that every descriptor's defaults validate.
9
10use crate::benchmark::BenchmarkDescriptor;
11use crate::benchmarks::{
12    agentic, bfcl, concurrency, contamination, decode_floor, default_tier_boot, kat_equality,
13    mlperf_agentic, quick_speed, scheduler_equivalence, serve_matrix, spec_cost, ssm_poison, ttft,
14    video, vision,
15};
16
17/// 2026-09-26: Every benchmark, in list order. A compile-time table of
18/// `&'static` descriptors: registration is a compile-time decision.
19const ALL: &[&BenchmarkDescriptor] = &[
20    // 2026-09-26: A measurement tool with no baseline and no thresholds,
21    // excused from the PR gate set in `gate::coverage::NOT_REQUIRED`.
22    &quick_speed::DESCRIPTOR,
23    // 2026-09-26: The gate counterpart of the probe above, judged against a
24    // BENCH.toml floor and listed in `gate::coverage::REQUIRED`.
25    &decode_floor::DESCRIPTOR,
26    // 2026-10-04: A measurement tool for the speculative cost model, excused from the PR gate
27    // set in `gate::coverage::NOT_REQUIRED`.
28    &spec_cost::DESCRIPTOR,
29    &concurrency::DESCRIPTOR,
30    &concurrency::DFLASH2_DESCRIPTOR,
31    // 2026-09-26: The MoE flagship's concurrency ladder, listed in
32    // `gate::coverage::REQUIRED`; see `concurrency::MOE_DESCRIPTOR`.
33    &concurrency::MOE_DESCRIPTOR,
34    &ttft::WARM_DESCRIPTOR,
35    &ttft::COLD_DESCRIPTOR,
36    // 2026-09-27: Listed in `gate::coverage::REQUIRED`: the 32k-prompt pair on
37    // the dense flagship, then on the 35B MoE.
38    &ttft::HIGH_ISL_COLD_DESCRIPTOR,
39    &ttft::HIGH_ISL_WARM_DESCRIPTOR,
40    &ttft::HIGH_ISL_COLD_MOE_DESCRIPTOR,
41    &ttft::HIGH_ISL_WARM_MOE_DESCRIPTOR,
42    &contamination::DESCRIPTOR,
43    // 2026-09-26: Listed in `gate::coverage::REQUIRED`. Coverage has no
44    // per-model dimension: the checkpoints it measures are the BENCH.toml
45    // entries that declare `gate = "vision-fidelity"`
46    // (`gate::bench::baseline_for`).
47    &vision::DESCRIPTOR,
48    // 2026-09-26: The same, for the entries that declare
49    // `gate = "video-fidelity"`.
50    &video::DESCRIPTOR,
51    &ssm_poison::DESCRIPTOR,
52    // 2026-09-26: Beside the poisoning gate: both check whether one request's
53    // state reaches another. That one asks whether a replay changes; this one
54    // whether the order matters.
55    &kat_equality::DESCRIPTOR,
56    // 2026-10-03: The cheap correctness suite of the Nemotron-3 models, one gate id per model
57    // (`kat_equality::nemotron`, `bfcl::mini_descriptors`). Listed in
58    // `gate::coverage::PROMOTION_CANDIDATES` until their BENCH entries are measured.
59    &kat_equality::NEMOTRON_NANO_DESCRIPTOR,
60    &kat_equality::NEMOTRON_SUPER_DESCRIPTOR,
61    &bfcl::MINI_NEMOTRON_NANO_DESCRIPTOR,
62    &bfcl::MINI_NEMOTRON_SUPER_DESCRIPTOR,
63    // 2026-10-03: The mini draw itself, a measurement draw (`gate::coverage::NOT_REQUIRED`).
64    &bfcl::SUBSET_MINI_DESCRIPTOR,
65    // 2026-09-26: The same family: does the router matter? The same draw
66    // under the synchronous and the asynchronous device router. Listed in
67    // `gate::coverage::PROMOTION_CANDIDATES`.
68    &scheduler_equivalence::DESCRIPTOR,
69    // 2026-10-01: The serve's memory plan at its default precision tier, read after one
70    // warmup wave. Listed in `gate::coverage::PROMOTION_CANDIDATES`.
71    &default_tier_boot::DESCRIPTOR,
72    &agentic::DESCRIPTOR,
73    &bfcl::SUBSET_DESCRIPTOR,
74    &bfcl::SUBSET_ECHOLP_DESCRIPTOR,
75    &bfcl::FULL_DESCRIPTOR,
76    // 2026-09-26: The two subset gates above are also benchmark groups
77    // (`gate::group::GROUPS`): a shard is the group's benchmark run with
78    // `--param shard=i/n`, not a benchmark of its own. The MLPerf entry below
79    // is unrunnable (its dataset is not published) and is listed so the pane
80    // shows it exists.
81    &mlperf_agentic::SUBSET_DESCRIPTOR,
82    // 2026-09-26: Last: it replaces the model the box is serving (its
83    // `detail`), so it sits furthest from an accidental start.
84    &serve_matrix::DESCRIPTOR,
85];
86
87pub fn all() -> &'static [&'static BenchmarkDescriptor] {
88    ALL
89}
90
91/// 2026-09-26: Look one up by its stable id.
92pub fn find(id: &str) -> Option<&'static BenchmarkDescriptor> {
93    all().iter().copied().find(|d| d.id == id)
94}
95
96#[cfg(test)]
97mod tests {
98    use super::*;
99
100    #[test]
101    fn ids_are_unique_and_filename_safe() {
102        let mut seen = std::collections::BTreeSet::new();
103        for d in all() {
104            assert!(!d.id.is_empty(), "benchmark ids address history files");
105            assert!(seen.insert(d.id), "duplicate benchmark id {}", d.id);
106            assert!(
107                d.id.chars()
108                    .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-'),
109                "{} is not filename-safe",
110                d.id
111            );
112        }
113    }
114
115    #[test]
116    fn find_round_trips_every_descriptor() {
117        for d in all() {
118            assert_eq!(find(d.id).unwrap().name, d.name);
119        }
120        assert!(find("nope").is_none());
121    }
122
123    /// 2026-09-26: A zero `expected_secs` would make the certification
124    /// planner (`bench_certify/plan.rs`) treat a runnable benchmark as free.
125    /// Zero is permitted only for benchmarks whose hint says they cannot run.
126    #[test]
127    fn every_runnable_benchmark_declares_an_expected_duration() {
128        for d in all() {
129            if d.duration_hint.starts_with("unrunnable") {
130                assert_eq!(
131                    d.expected_secs, 0,
132                    "{}: unrunnable but has a duration",
133                    d.id
134                );
135                continue;
136            }
137            assert!(d.expected_secs > 0, "{}: expected_secs is 0", d.id);
138        }
139        // 2026-09-26: Pin the required gates and the groups by name, so a new
140        // entry cannot slip in with the field left at zero.
141        for id in crate::gate::REQUIRED_GATES {
142            assert!(find(id).unwrap().expected_secs > 0, "{id}");
143        }
144        for g in crate::gate::group::GROUPS {
145            assert!(find(g.id).unwrap().expected_secs > 0, "{}", g.id);
146        }
147    }
148
149    #[test]
150    fn every_benchmark_declares_defaults_that_validate() {
151        for d in all() {
152            let b = d.build();
153            let specs = b.parameters();
154            let values = crate::params::ParamValues::defaults(&specs);
155            values
156                .validate_against(&specs)
157                .unwrap_or_else(|e| panic!("{}: {e}", d.id));
158        }
159    }
160}
metrale_bench/registry.rs

metrale_bench/
registry.rs
