metrale_bench/benchmarks/bfcl/
report.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: How a BFCL run is presented (table, summary tiles, gate
4//! metrics) and its run verdict, including the MLPerf-edge floors.
5//!
6//! Owner: bench, BFCL benchmark.
7//! Invariants: none beyond the types.
8
9use std::collections::BTreeMap;
10
11use anyhow::Result;
12
13use super::Bfcl;
14use crate::benchmarks::bfcl::draw;
15use crate::params::{ParamKind, ParamSpec, ParamValue, ParamValues};
16use crate::result::{Cell, CellStyle, Column, ResultTable, Stat, Verdict};
17
18/// 2026-09-26: MLPerf-edge floors for Qwen3.6-27B: the llama.cpp Q4_K_M
19/// reference (86.23 / 87.96, recorded in `kernels/gb10/qwen3.6-27b/BENCH.toml`)
20/// times 0.97.
21pub const MLPERF_FLOOR_OVERALL: f64 = 83.64;
22pub const MLPERF_FLOOR_NORMALIZED: f64 = 85.32;
23
24/// 2026-09-26: The only served models whose run verdict is gated on the MLPerf
25/// floors (`floor_verdict`). The summary tiles style every model against the
26/// floors.
27pub const MLPERF_FLOOR_CHECKPOINTS: [&str; 2] = [
28    "unsloth/Qwen3.6-27B-NVFP4",
29    "centml/Qwen3.6-27B-NVFP4-W4A4-mlpinf",
30];
31
32/// 2026-09-26: Compared ignoring ASCII case, so a serve that spells the org
33/// differently still matches.
34pub fn is_mlperf_submission_checkpoint(model: &str) -> bool {
35    MLPERF_FLOOR_CHECKPOINTS
36        .iter()
37        .any(|c| c.eq_ignore_ascii_case(model))
38}
39
40/// 2026-09-26: The run-verdict floors for a model outside
41/// [`MLPERF_FLOOR_CHECKPOINTS`]. Under `--pull-request-gate` the server's
42/// `apply_threshold_params` fills them from the model's BENCH.toml `min`
43/// bounds minus their noise. 0.0 on both means not gating: the run keeps an
44/// info verdict.
45#[derive(Clone, Copy, Debug, Default, PartialEq)]
46pub(super) struct BaselineMins {
47    pub overall: f64,
48    pub normalized: f64,
49}
50
51impl BaselineMins {
52    // 2026-09-26: The 0.0 defaults are the documented off state (see the
53    // parameter help), not an implicit bar.
54    pub(super) fn specs() -> [ParamSpec; 2] {
55        const PCT: ParamKind = ParamKind::Float {
56            min: 0.0,
57            max: 100.0,
58        };
59        [
60            ParamSpec::new(
61                "min_overall",
62                "Overall floor",
63                "Run-verdict floor on overall_accuracy. 0 disables (a standalone run reports \
64                 an info verdict); under --pull-request-gate this is auto-filled from the \
65                 selected variant's BENCH.toml `min` bound. Ignored on the MLPerf submission \
66                 checkpoints, which keep the MLPerf floor verdict.",
67                PCT,
68                ParamValue::Float(0.0),
69            ),
70            ParamSpec::new(
71                "min_normalized",
72                "Normalized floor",
73                "Run-verdict floor on normalized_single_turn_score. 0 disables (a standalone \
74                 run reports an info verdict); under --pull-request-gate this is auto-filled \
75                 from the selected variant's BENCH.toml `min` bound. Ignored on the MLPerf \
76                 submission checkpoints, which keep the MLPerf floor verdict.",
77                PCT,
78                ParamValue::Float(0.0),
79            ),
80        ]
81    }
82
83    pub(super) fn from_values(values: &ParamValues) -> Result<Self> {
84        Ok(Self {
85            overall: values.float("min_overall")?,
86            normalized: values.float("min_normalized")?,
87        })
88    }
89
90    fn gating(self) -> bool {
91        self.overall > 0.0 || self.normalized > 0.0
92    }
93}
94
95impl Bfcl {
96    pub(super) fn table(&self) -> Option<ResultTable> {
97        let scores = self.scores.as_ref()?;
98        let mut t = ResultTable::new(
99            "PER-SUBSET ACCURACY",
100            vec![
101                Column::left("Subset", 24),
102                Column::left("Category", 14),
103                Column::right("accuracy %", 11),
104            ],
105        );
106        for (subset, value) in &scores.subset_scores {
107            t.push(vec![
108                Cell::new(subset.clone()),
109                Cell::styled(
110                    draw::category_of(subset).unwrap_or("unscored").to_string(),
111                    CellStyle::Dim,
112                ),
113                Cell::styled(
114                    format!("{value:.2}"),
115                    match *value {
116                        v if v >= 90.0 => CellStyle::Good,
117                        v if v >= 60.0 => CellStyle::Neutral,
118                        _ => CellStyle::Warn,
119                    },
120                ),
121            ]);
122        }
123        for (category, value) in &scores.category_scores {
124            t.push(vec![
125                Cell::styled(format!("▸ {category}"), CellStyle::Accent),
126                Cell::styled("category".to_string(), CellStyle::Dim),
127                Cell::styled(format!("{value:.2}"), CellStyle::Accent),
128            ]);
129        }
130        Some(t)
131    }
132
133    pub(super) fn summary(&self) -> Vec<Stat> {
134        match &self.scores {
135            Some(s) => vec![
136                Stat::new(
137                    "Overall accuracy",
138                    format!("{:.2}", s.overall_accuracy),
139                    "%",
140                )
141                .with_style(floor_style(s.overall_accuracy, MLPERF_FLOOR_OVERALL)),
142                Stat::new(
143                    "Normalized single-turn",
144                    format!("{:.2}", s.normalized_single_turn_score),
145                    "%",
146                )
147                .with_style(floor_style(
148                    s.normalized_single_turn_score,
149                    MLPERF_FLOOR_NORMALIZED,
150                )),
151                Stat::new("Samples", s.total_samples.to_string(), ""),
152            ],
153            None => vec![
154                Stat::new(
155                    "Samples",
156                    format!("{}/{}", self.cursor, self.samples.len()),
157                    "",
158                ),
159                Stat::new("With tool calls", self.tool_call_samples.to_string(), ""),
160            ],
161        }
162    }
163
164    /// 2026-09-26: The gate metrics, from the same scores the summary tiles
165    /// read. Empty until scoring completes.
166    pub(super) fn metrics(&self) -> BTreeMap<String, f64> {
167        let Some(s) = &self.scores else {
168            return BTreeMap::new();
169        };
170        let mut m = BTreeMap::new();
171        m.insert("overall_accuracy".to_string(), s.overall_accuracy);
172        m.insert(
173            "normalized_single_turn_score".to_string(),
174            s.normalized_single_turn_score,
175        );
176        m.insert("samples".to_string(), s.total_samples as f64);
177        // 2026-09-26: Per-subset counts, so a group can aggregate its shards
178        // (`aggregate::tallies_from_metrics`). Counts, not scores: a mean of
179        // shard scores is not the whole-set value (see `aggregate`).
180        // `check_record` looks up only the metrics the baseline entry names.
181        for (subset, (hits, n)) in &s.subset_totals {
182            m.insert(format!("subset.{subset}.hits"), *hits as f64);
183            m.insert(format!("subset.{subset}.n"), *n as f64);
184        }
185        // 2026-09-26: Always emitted, so 0 is a measurement rather than an
186        // absent key.
187        m.insert("transport_errors".to_string(), self.transport_errors as f64);
188        // 2026-09-26: How many samples from `sensitive::KNOWN_PARTITION_SENSITIVE`
189        // this run collected. Always emitted.
190        m.insert(
191            "known_partition_sensitive".to_string(),
192            self.known_sensitive_seen as f64,
193        );
194        // 2026-09-26: Which shard this run measured, from the run itself.
195        // `GateRecord::shard` (gate/record.rs) reads these two keys.
196        if let Some(shard) = self.shard {
197            m.insert("shard.index".to_string(), shard.index as f64);
198            m.insert("shard.count".to_string(), shard.count as f64);
199        }
200        m
201    }
202
203    pub(super) fn verdict(&self) -> Verdict {
204        let Some(s) = &self.scores else {
205            return Verdict::info("not scored");
206        };
207        // 2026-09-26: A shard's slice is not judged against floors cut for the
208        // whole draw. It reports an info verdict, and `gate::check_group`
209        // judges the aggregate of the partition's per-subset counts.
210        if let Some(shard) = self.shard {
211            return Verdict::info(format!(
212                "shard {}/{} — overall {:.2} · normalized {:.2} · n={} on this slice; the \
213                 group's verdict is the aggregate over the whole partition",
214                shard.index,
215                shard.count,
216                s.overall_accuracy,
217                s.normalized_single_turn_score,
218                s.total_samples
219            ));
220        }
221        floor_verdict(self.target_model.as_deref(), s, self.baseline_mins)
222    }
223}
224
225/// 2026-09-26: The run verdict, scoped by served model. On
226/// [`MLPERF_FLOOR_CHECKPOINTS`] the MLPerf floors give PASS or FAIL. Otherwise
227/// nonzero `mins` give PASS or FAIL against them, and without them the verdict
228/// is info. An unknown model is treated as a non-submission model. Only PASS
229/// satisfies `GateRecord::verdict_passes`.
230fn floor_verdict(target_model: Option<&str>, s: &super::Scores, mins: BaselineMins) -> Verdict {
231    let detail = format!(
232        "overall {:.2} (floor {MLPERF_FLOOR_OVERALL}) · normalized {:.2} (floor \
233         {MLPERF_FLOOR_NORMALIZED}) · n={}",
234        s.overall_accuracy, s.normalized_single_turn_score, s.total_samples
235    );
236    match target_model {
237        Some(m) if is_mlperf_submission_checkpoint(m) => {
238            let overall_ok = s.overall_accuracy >= MLPERF_FLOOR_OVERALL;
239            let normalized_ok = s.normalized_single_turn_score >= MLPERF_FLOOR_NORMALIZED;
240            if overall_ok && normalized_ok {
241                Verdict::pass(detail)
242            } else {
243                Verdict::fail(format!("BELOW THE MLPERF-EDGE FLOOR — {detail}"))
244            }
245        }
246        // 2026-09-26: Raw value against the bar as handed in. When the gate
247        // fills the bar, `apply_threshold_params` has already subtracted the
248        // bound's noise, so this agrees with `gate::scoring::compare`.
249        _ if mins.gating() => {
250            let bars = format!(
251                "overall {:.2} (baseline min {:.2}) · normalized {:.2} (baseline min {:.2}) \
252                 · n={}",
253                s.overall_accuracy,
254                mins.overall,
255                s.normalized_single_turn_score,
256                mins.normalized,
257                s.total_samples
258            );
259            if s.overall_accuracy >= mins.overall
260                && s.normalized_single_turn_score >= mins.normalized
261            {
262                Verdict::pass(format!("{bars} — clears this checkpoint's committed bars"))
263            } else {
264                Verdict::fail(format!("BELOW THE BASELINE THRESHOLDS — {bars}"))
265            }
266        }
267        Some(m) => Verdict::info(format!(
268            "{detail} — judged by baseline thresholds: {m} is not an MLPerf submission \
269             checkpoint, so the floor does not gate this run (it does not transfer \
270             across weights); the floor styling above is a visual reference only"
271        )),
272        None => Verdict::info(format!(
273            "{detail} — judged by baseline thresholds: served model unknown, so the \
274             MLPerf floor (defined on the Qwen3.6-27B submission checkpoints) does \
275             not gate this run"
276        )),
277    }
278}
279
280fn floor_style(value: f64, floor: f64) -> CellStyle {
281    if value >= floor {
282        CellStyle::Good
283    } else {
284        CellStyle::Bad
285    }
286}
metrale_bench/benchmarks/bfcl/report.rs

metrale_bench/benchmarks/bfcl/
report.rs
