metrale_bench/benchmarks/spec_cost/
acceptance.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-10-04: The run's acceptance counts, for the drafter's confidence→acceptance
4//! calibration: between the probe's `/metrics` page and the one after the last width, the
5//! change in
6//! - `metrale_spec_draft_confidence_total{le, accepted}`: drafts a verify step reached, by
7//!   the drafter's top-1 log-probability bucket (upper edge `le`) and verdict; and
8//! - `metrale_spec_verify_steps_total{drafts, accepted}`: verify steps by drafts checked and
9//!   accepted, for the per-position priors.
10//!
11//! Both are whole-run totals, not per-width: acceptance is a property of the drafter and the
12//! text, while the cost cells are per width. Pure: no I/O.
13//!
14//! Owner: bench, spec-cost.
15//! Invariants:
16//! - A series absent from the first page counts from 0 (the serve renders them once they
17//!   fire); one that went backwards fails the run (the serve restarted).
18//! - The record keys are written by `record` and read by [`read`] only.
19
20use std::collections::BTreeMap;
21
22use anyhow::{Context, Result, bail};
23
24use super::prom::Scrape;
25
26const CONFIDENCE_METRIC: &str = "metrale_spec_draft_confidence_total";
27const STEPS_METRIC: &str = "metrale_spec_verify_steps_total";
28
29/// 2026-10-04: The run's acceptance counts, read back from a record.
30#[derive(Clone, Debug, Default, PartialEq)]
31pub struct AcceptanceCounts {
32    /// 2026-10-04: Bucket upper edges, ascending, as the serve labelled them.
33    pub edges: Vec<f32>,
34    pub accepted: Vec<u64>,
35    pub rejected: Vec<u64>,
36    /// 2026-10-04: `(drafts, accepted)` → verify steps.
37    pub steps: BTreeMap<(usize, usize), u64>,
38}
39
40fn delta(first: &Scrape, last: f64, name: &str, labels: &[(&str, &str)]) -> Result<u64> {
41    let before = first.value(name, labels)?.unwrap_or(0.0);
42    let d = last - before;
43    if d < 0.0 || !d.is_finite() {
44        bail!("{name}{labels:?} went from {before} to {last}: the serve restarted");
45    }
46    Ok(d as u64)
47}
48
49/// 2026-10-04: Writes the counts between `first` and `last` into `m`: per bucket `i`
50/// (ascending edge) `conf{i}_le`, `conf{i}_accepted`, `conf{i}_rejected`; per verify shape
51/// `steps_d{d}_a{a}`.
52pub(crate) fn record(first: &Scrape, last: &Scrape, m: &mut BTreeMap<String, f64>) -> Result<()> {
53    let mut buckets: BTreeMap<String, (f32, [u64; 2])> = BTreeMap::new();
54    for s in last.series.iter().filter(|s| s.name == CONFIDENCE_METRIC) {
55        let (Some(le), Some(acc)) = (s.labels.get("le"), s.labels.get("accepted")) else {
56            bail!(
57                "{CONFIDENCE_METRIC} without le/accepted labels: {:?}",
58                s.labels
59            );
60        };
61        let edge: f32 = le
62            .parse()
63            .with_context(|| format!("{CONFIDENCE_METRIC} le={le:?}"))?;
64        let slot = match acc.as_str() {
65            "1" => 1,
66            "0" => 0,
67            other => bail!("{CONFIDENCE_METRIC} accepted={other:?}"),
68        };
69        let d = delta(
70            first,
71            s.value,
72            CONFIDENCE_METRIC,
73            &[("le", le), ("accepted", acc)],
74        )?;
75        buckets.entry(le.clone()).or_insert((edge, [0, 0])).1[slot] = d;
76    }
77    let mut ordered: Vec<(f32, [u64; 2])> = buckets.into_values().collect();
78    ordered.sort_by(|a, b| a.0.total_cmp(&b.0));
79    for (i, (edge, [rej, acc])) in ordered.into_iter().enumerate() {
80        m.insert(format!("conf{i}_le"), f64::from(edge));
81        m.insert(format!("conf{i}_accepted"), acc as f64);
82        m.insert(format!("conf{i}_rejected"), rej as f64);
83    }
84    for s in last.series.iter().filter(|s| s.name == STEPS_METRIC) {
85        let (Some(d), Some(a)) = (s.labels.get("drafts"), s.labels.get("accepted")) else {
86            bail!(
87                "{STEPS_METRIC} without drafts/accepted labels: {:?}",
88                s.labels
89            );
90        };
91        let n = delta(
92            first,
93            s.value,
94            STEPS_METRIC,
95            &[("drafts", d), ("accepted", a)],
96        )?;
97        m.insert(format!("steps_d{d}_a{a}"), n as f64);
98    }
99    Ok(())
100}
101
102/// 2026-10-04: The counts `record` wrote; `None` when the run saw no draft confidence (a
103/// `k = 0` serve, or one that never batched a verify).
104pub fn read(metrics: &BTreeMap<String, f64>) -> Result<Option<AcceptanceCounts>> {
105    let mut c = AcceptanceCounts::default();
106    while let Some(&le) = metrics.get(&format!("conf{}_le", c.edges.len())) {
107        let i = c.edges.len();
108        let get = |f: &str| {
109            metrics
110                .get(&format!("conf{i}_{f}"))
111                .copied()
112                .with_context(|| format!("acceptance record: bucket {i} has no {f}"))
113        };
114        c.accepted.push(get("accepted")? as u64);
115        c.rejected.push(get("rejected")? as u64);
116        c.edges.push(le as f32);
117    }
118    if c.edges.is_empty() {
119        return Ok(None);
120    }
121    for (key, &v) in metrics {
122        let Some(rest) = key.strip_prefix("steps_d") else {
123            continue;
124        };
125        let parsed = rest
126            .split_once("_a")
127            .and_then(|(d, a)| Some((d.parse().ok()?, a.parse().ok()?)));
128        let shape = parsed.with_context(|| format!("acceptance record: bad key {key:?}"))?;
129        c.steps.insert(shape, v as u64);
130    }
131    Ok(Some(c))
132}
133
134#[cfg(test)]
135#[path = "acceptance_tests.rs"]
136mod tests;
metrale_bench/benchmarks/spec_cost/acceptance.rs

metrale_bench/benchmarks/spec_cost/
acceptance.rs
