metrale_bench/gate/
card.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Shareable result cards: a benchmark record rendered onto the SVG card template
4//! (`assets/cards/result-card.svg`).
5//!
6//! Owner: bench gate.
7//! Invariants:
8//! - One template for every benchmark: [`spec_for`] maps a benchmark's metrics onto one hero
9//!   number and four detail slots.
10//! - Every card prints the model, quantization, recipe, benchmark id and hardware beside the
11//!   number; an unknown quantization or recipe renders as `—`.
12use crate::gate::record::GateRecord;
13use std::collections::BTreeMap;
14
15/// 2026-09-26: How a raw `f64` becomes card text.
16#[derive(Clone, Copy, Debug)]
17pub enum Fmt {
18    /// 2026-09-26: Rounded to an integer: `115`.
19    Int,
20    /// 2026-09-26: One decimal: `22.7`.
21    One,
22    /// 2026-09-26: Two decimals: `84.22`.
23    Two,
24    /// 2026-09-26: Rounded milliseconds: `8288 ms`.
25    Ms,
26}
27
28impl Fmt {
29    pub fn apply(self, v: f64) -> String {
30        match self {
31            Fmt::Int => format!("{}", v.round() as i64),
32            Fmt::One => format!("{v:.1}"),
33            Fmt::Two => format!("{v:.2}"),
34            Fmt::Ms => format!("{} ms", v.round() as i64),
35        }
36    }
37}
38
39/// 2026-09-26: One of the four detail boxes: its label, the metric key and the format.
40#[derive(Clone, Copy, Debug)]
41pub struct Slot {
42    pub label: &'static str,
43    pub key: &'static str,
44    pub fmt: Fmt,
45}
46
47const fn slot(label: &'static str, key: &'static str, fmt: Fmt) -> Option<Slot> {
48    Some(Slot { label, key, fmt })
49}
50
51/// 2026-09-26: What a benchmark puts on a card.
52#[derive(Clone, Debug)]
53pub struct CardSpec {
54    pub hero_label: &'static str,
55    pub hero_key: &'static str,
56    pub hero_note: &'static str,
57    pub hero_fmt: Fmt,
58    pub slots: [Option<Slot>; 4],
59}
60
61/// 2026-09-26: The per-benchmark mapping. Its keys are metric names that committed records
62/// carry: `every_card_key_exists_on_a_real_record` fails if the last record file (by name)
63/// of a gate lacks one, since a missing key hides its box.
64pub fn spec_for(benchmark_id: &str) -> CardSpec {
65    match benchmark_id {
66        "decode-floor" => CardSpec {
67            hero_label: "Tokens / sec",
68            hero_key: "server_decode_tok_s",
69            hero_note: "decode, steady state",
70            hero_fmt: Fmt::One,
71            slots: [
72                slot("Accept length", "accept_len_mean", Fmt::Two),
73                slot("Output tokens", "output_tokens", Fmt::Int),
74                slot("Runs", "runs", Fmt::Int),
75                None,
76            ],
77        },
78        "concurrency-sweep" => CardSpec {
79            hero_label: "Tokens / sec",
80            hero_key: "peak_aggregate_tok_s",
81            hero_note: "aggregate, best rung of C=1..128",
82            hero_fmt: Fmt::One,
83            slots: [
84                slot("C=1", "c1_aggregate_tok_s", Fmt::One),
85                slot("C=16", "c16_aggregate_tok_s", Fmt::One),
86                slot("C=128", "c128_aggregate_tok_s", Fmt::One),
87                slot("TTFT p50, C=1", "c1_ttft_p50_ms", Fmt::Ms),
88            ],
89        },
90        // 2026-09-26: This gate's BENCH.toml entry runs C=1..16 (batch cap 16), so there is
91        // no C=128 slot to show.
92        "concurrency-sweep-dflash2" => CardSpec {
93            hero_label: "Tokens / sec",
94            hero_key: "peak_aggregate_tok_s",
95            hero_note: "aggregate, best rung of C=1..16, DFlash2 armed",
96            hero_fmt: Fmt::One,
97            slots: [
98                slot("C=1", "c1_aggregate_tok_s", Fmt::One),
99                slot("C=4", "c4_aggregate_tok_s", Fmt::One),
100                slot("C=16", "c16_aggregate_tok_s", Fmt::One),
101                slot("TTFT p50, C=1", "c1_ttft_p50_ms", Fmt::Ms),
102            ],
103        },
104        // 2026-09-26: This gate's BENCH.toml entry pins C=1..16, ISL 128, OSL 1024, so there is
105        // no C=128 slot to show.
106        "concurrency-sweep-moe" => CardSpec {
107            hero_label: "Tokens / sec",
108            hero_key: "peak_aggregate_tok_s",
109            hero_note: "aggregate, best rung of C=1..16, ISL 128 / OSL 1024",
110            hero_fmt: Fmt::One,
111            slots: [
112                slot("C=1", "c1_aggregate_tok_s", Fmt::One),
113                slot("C=4", "c4_aggregate_tok_s", Fmt::One),
114                slot("C=16", "c16_aggregate_tok_s", Fmt::One),
115                slot("TTFT p50, C=1", "c1_ttft_p50_ms", Fmt::Ms),
116            ],
117        },
118        "bfcl-subset" | "bfcl-subset-echolp" => CardSpec {
119            hero_label: "Overall accuracy",
120            hero_key: "overall_accuracy",
121            hero_note: "BFCL single-turn — see samples for the draw",
122            hero_fmt: Fmt::Two,
123            slots: [
124                slot("Normalized ST", "normalized_single_turn_score", Fmt::Two),
125                slot("Samples", "samples", Fmt::Int),
126                None,
127                None,
128            ],
129        },
130        "agentic-webserver" => CardSpec {
131            hero_label: "Webserver OK",
132            hero_key: "webserver_ok",
133            hero_note: "runs that built, served and tore down cleanly",
134            hero_fmt: Fmt::Int,
135            slots: [
136                slot("Followed directions", "followed_directions", Fmt::Int),
137                slot("Iterations", "iterations", Fmt::Int),
138                slot("Seconds / turn", "s_per_turn", Fmt::Two),
139                slot("Decode tok/s", "decode_tps", Fmt::One),
140            ],
141        },
142        "ttft-cold-gate" => CardSpec {
143            hero_label: "TTFT median",
144            hero_key: "median_ms",
145            hero_note: "cold, first token after load",
146            hero_fmt: Fmt::Ms,
147            slots: [
148                slot("p90", "p90_ms", Fmt::Ms),
149                slot("Samples", "samples", Fmt::Int),
150                None,
151                None,
152            ],
153        },
154        "ttft-warm-gate" => CardSpec {
155            hero_label: "TTFT median",
156            hero_key: "median_ms",
157            hero_note: "warm, prefix cache primed",
158            hero_fmt: Fmt::Ms,
159            slots: [
160                slot("p90", "p90_ms", Fmt::Ms),
161                slot("Samples", "samples", Fmt::Int),
162                None,
163                None,
164            ],
165        },
166        // 2026-09-27: One-shot gates: the one sample is the median and the p90, so the card
167        // shows it once, beside the prompt size the server counted.
168        "high-isl-ttft-cold" | "high-isl-ttft-cold-moe" => CardSpec {
169            hero_label: "TTFT, 32k prompt",
170            hero_key: "median_ms",
171            hero_note: "cold, one fresh request",
172            hero_fmt: Fmt::Ms,
173            slots: [
174                slot("Prompt tokens", "prompt_tokens", Fmt::Int),
175                slot("Samples", "samples", Fmt::Int),
176                None,
177                None,
178            ],
179        },
180        "high-isl-ttft-warm" | "high-isl-ttft-warm-moe" => CardSpec {
181            hero_label: "TTFT, 32k prompt",
182            hero_key: "median_ms",
183            hero_note: "warm, the same request re-sent",
184            hero_fmt: Fmt::Ms,
185            slots: [
186                slot("Prompt tokens", "prompt_tokens", Fmt::Int),
187                slot("Samples", "samples", Fmt::Int),
188                None,
189                None,
190            ],
191        },
192        "vision-fidelity" => CardSpec {
193            hero_label: "Geometry cells matched",
194            hero_key: "geometry_matched",
195            hero_note: "vision fidelity, control held",
196            hero_fmt: Fmt::Int,
197            slots: [
198                slot("Cells asserted", "geometry_asserted", Fmt::Int),
199                slot("Probes passed", "probes_passed", Fmt::Int),
200                slot("Probes total", "probes_total", Fmt::Int),
201                None,
202            ],
203        },
204        "video-fidelity" => CardSpec {
205            hero_label: "Legs passed",
206            hero_key: "legs_passed",
207            hero_note: "video fidelity, control held",
208            hero_fmt: Fmt::Int,
209            slots: [
210                slot("Legs asserted", "legs_asserted", Fmt::Int),
211                slot("Skipped", "legs_skipped", Fmt::Int),
212                None,
213                None,
214            ],
215        },
216        "ssm-state-poisoning-gate" => CardSpec {
217            hero_label: "Replays byte-identical",
218            hero_key: "invariant",
219            hero_note: "SSM state isolation under interleaving",
220            hero_fmt: Fmt::Int,
221            slots: [
222                slot("Rounds", "rounds", Fmt::Int),
223                slot("Collapsed", "collapsed", Fmt::Int),
224                slot("Jittered", "jittered", Fmt::Int),
225                None,
226            ],
227        },
228        // 2026-09-26: Unknown benchmark: the hero is the alphabetically first metric (the
229        // metrics are a `BTreeMap`), so the same record always renders the same card.
230        _ => CardSpec {
231            hero_label: "Result",
232            hero_key: "",
233            hero_note: "first metric, alphabetically — no card mapping for this benchmark yet",
234            hero_fmt: Fmt::Two,
235            slots: [None, None, None, None],
236        },
237    }
238}
239
240/// 2026-09-26: `author=Ada,website=ada.dev` -> map, with keys lowercased. Later keys win; spaces
241/// and empty pairs are ignored. A part without `=`, or with an empty key, is an error rather
242/// than dropped, so a typo cannot silently leave a field off the card.
243pub fn parse_args(raw: &str) -> Result<BTreeMap<String, String>, String> {
244    let mut out = BTreeMap::new();
245    for part in raw.split(',') {
246        let p = part.trim();
247        if p.is_empty() {
248            continue;
249        }
250        let (k, v) = p
251            .split_once('=')
252            .ok_or_else(|| format!("`{p}` is not key=value"))?;
253        let k = k.trim();
254        if k.is_empty() {
255            return Err(format!("`{p}` has an empty key"));
256        }
257        out.insert(k.to_lowercase(), v.trim().to_string());
258    }
259    Ok(out)
260}
261
262/// 2026-09-26: Escapes `&`, `<` and `>` for XML text; an unescaped `&` in a model id makes the
263/// SVG invalid.
264fn esc(s: &str) -> String {
265    s.replace('&', "&amp;")
266        .replace('<', "&lt;")
267        .replace('>', "&gt;")
268}
269
270/// 2026-09-26: Set an attribute on the element carrying `id`, replacing it if present; no-op
271/// when the id is absent. Used to hide a whole box with `display="none"` on its group.
272fn set_attr(svg: &mut String, id: &str, attr: &str, value: &str) {
273    let needle = format!("id=\"{id}\"");
274    let Some(at) = svg.find(&needle) else { return };
275    let Some(open) = svg[..at].rfind('<') else {
276        return;
277    };
278    let Some(close) = svg[at..].find('>').map(|i| at + i) else {
279        return;
280    };
281    let existing = format!(" {attr}=\"");
282    if let Some(k) = svg[open..close].find(&existing) {
283        let vstart = open + k + existing.len();
284        let Some(vend) = svg[vstart..close].find('"').map(|i| vstart + i) else {
285            return;
286        };
287        svg.replace_range(vstart..vend, value);
288        return;
289    }
290    svg.insert_str(close, &format!(" {attr}=\"{value}\""));
291}
292
293/// 2026-09-26: Replace the text body of `<... id="ID" ...>body</...>` with `value`, XML-escaped;
294/// no-op when the id is absent.
295fn set(svg: &mut String, id: &str, value: &str) {
296    let needle = format!("id=\"{id}\"");
297    let Some(at) = svg.find(&needle) else { return };
298    let Some(gt) = svg[at..].find('>').map(|i| at + i + 1) else {
299        return;
300    };
301    let Some(lt) = svg[gt..].find('<').map(|i| gt + i) else {
302        return;
303    };
304    svg.replace_range(gt..lt, &esc(value));
305}
306
307/// 2026-09-26: Render a record onto the card template.
308pub fn render(template: &str, record: &GateRecord, args: &BTreeMap<String, String>) -> String {
309    let spec = spec_for(&record.benchmark_id);
310    let mut svg = template.to_string();
311    let m = &record.metrics;
312
313    let hero = if spec.hero_key.is_empty() {
314        m.iter().next().map(|(k, v)| (k.clone(), *v))
315    } else {
316        m.get(spec.hero_key)
317            .map(|v| (spec.hero_key.to_string(), *v))
318    };
319    match hero {
320        Some((key, v)) => {
321            set(&mut svg, "value-toks", &spec.hero_fmt.apply(v));
322            let label = if spec.hero_key.is_empty() {
323                key
324            } else {
325                spec.hero_label.to_string()
326            };
327            set(&mut svg, "label-toks", &label);
328        }
329        // 2026-09-26: No hero metric (a failed run): print `—`, not the template's `000.0`.
330        None => {
331            set(&mut svg, "value-toks", "—");
332            set(&mut svg, "label-toks", spec.hero_label);
333        }
334    }
335    set(&mut svg, "note-toks", spec.hero_note);
336
337    match record.verdict.as_deref() {
338        Some(_) if record.verdict_passes() => {
339            set(&mut svg, "value-verdict", "PASS");
340            set_attr(&mut svg, "verdict-chip", "fill", "#12B981");
341        }
342        Some(_) => {
343            set(&mut svg, "value-verdict", "FAIL");
344            set_attr(&mut svg, "verdict-chip", "fill", "#EFB338");
345            // 2026-09-26: Grey the hero so a failed run's number or dash does not read as a result.
346            set_attr(&mut svg, "value-toks", "fill", "#82868F");
347        }
348        // 2026-09-26: An ungated run has no verdict; hide the chip.
349        None => set_attr(&mut svg, "field-verdict", "display", "none"),
350    }
351
352    set(&mut svg, "value-model", &record.target_model);
353    set(
354        &mut svg,
355        "value-quant",
356        record
357            .params
358            .get("quant")
359            .or_else(|| record.params.get("quantization"))
360            .map_or_else(|| quant_from_model(&record.target_model), |q| q.clone())
361            .as_str(),
362    );
363    set(
364        &mut svg,
365        "value-recipe",
366        record.served_by.as_deref().unwrap_or("—"),
367    );
368    set(&mut svg, "value-test", &record.benchmark_id);
369    set(&mut svg, "value-hardware", &hardware_line(record));
370
371    for (i, s) in spec.slots.iter().enumerate() {
372        let (lid, vid) = (format!("label-m{}", i + 1), format!("value-m{}", i + 1));
373        match s.and_then(|s| m.get(s.key).map(|v| (s, *v))) {
374            Some((s, v)) => {
375                set(&mut svg, &lid, s.label);
376                set(&mut svg, &vid, &s.fmt.apply(v));
377            }
378            // 2026-09-26: Hide the whole box of an unused slot, rather than leave the template's
379            // placeholder text or an empty bordered box.
380            None => set_attr(&mut svg, &format!("field-m{}", i + 1), "display", "none"),
381        }
382    }
383
384    for (field, value_id, value) in [
385        ("field-author", "value-author", args.get("author")),
386        ("field-handle", "value-handle", args.get("handle")),
387        (
388            "field-site",
389            "value-site",
390            args.get("website").or_else(|| args.get("site")),
391        ),
392    ] {
393        match value {
394            Some(v) if !v.is_empty() => set(&mut svg, value_id, v),
395            _ => set_attr(&mut svg, field, "display", "none"),
396        }
397    }
398    svg
399}
400
401/// 2026-09-26: The quantization read off the checkpoint id (`unsloth/Qwen3.8-27B-NVFP4`) when
402/// the record has no `quant`/`quantization` param; `—` when no known tag appears.
403fn quant_from_model(model: &str) -> String {
404    let up = model.to_uppercase();
405    for tag in ["NVFP4", "FP8", "W4A4", "W4A16", "BF16", "INT8", "FP16"] {
406        if up.contains(tag) {
407            return tag.to_string();
408        }
409    }
410    "—".to_string()
411}
412
413fn hardware_line(record: &GateRecord) -> String {
414    let hw = &record.hardware;
415    let gpu = hw.gate_key();
416    match record
417        .hardware_state
418        .as_ref()
419        .and_then(|s| s.before.sm_clock_mhz)
420    {
421        Some(mhz) => format!("{gpu}, SM {mhz:.0} MHz"),
422        None => gpu,
423    }
424}
metrale_bench/gate/card.rs

metrale_bench/gate/
card.rs
