metrale_bench/benchmarks/decode_floor/
mod.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Decode Floor Gate: the pinned single-user decode throughput
4//! floor. Every generation knob is pinned so that two runs are comparable, and
5//! the one metric, the median server decode rate over `RUNS` runs, is judged
6//! against the served variant's BENCH.toml `server_decode_tok_s` bound under
7//! `--pull-request-gate`. `decode-floor` is listed in `gate::coverage::REQUIRED`.
8//!
9//! The pins (the benchmark's definition, not parameters):
10//! - Prompt: `MINHEAP_PROMPT`. A different prompt is a different benchmark.
11//! - Request: temperature 0, seed 0, `max_tokens` = `MAX_TOKENS`, and
12//!   `reasoning_effort: "none"` in the body, which the server maps to thinking
13//!   off, so the gate serve needs no thinking flag. Thinking off matters:
14//!   speculative dispatch is off inside `<think>` unless
15//!   `METRALE_DFLASH_SPEC_THINK=1` (`spec_dispatch_eligible`), so a
16//!   thinking-on run would measure serial decode.
17//! - Runs: `RUNS`, with no warmup parameter. The metric is the median
18//!   `usage."response_token/s"`.
19//!
20//! Vacuity pins (`score::evaluate`): the run is inconclusive, rendered as a
21//! failing verdict, unless every run emits at least `MIN_OUTPUT_TOKENS`,
22//! reports a finite positive server decode rate and a nonzero
23//! `usage.completion_tokens_details.accepted_prediction_tokens`, and the mean
24//! of the per-run accept lengths (`stats::accept_len`: emitted tokens per
25//! decode step, `completion / (completion - accepted)`) is at least
26//! `MIN_ACCEPT_LEN`.
27//!
28//! Owner: bench, decode_floor.
29//! Invariants: the run verdict is a pass only for `Evaluation::Measured` with
30//! `min_tok_s > 0` and a median at or above it (`verdict_for`).
31
32use crate::hardware::Sensitivity;
33use std::collections::BTreeMap;
34use std::future::Future;
35use std::time::{Duration, Instant};
36
37use anyhow::{Context, Result, bail};
38use serde_json::json;
39
40use crate::benchmark::{Benchmark, BenchmarkDescriptor, ModelExpectation};
41use crate::hardware::energy_sampler::EnergyMeter;
42use crate::http;
43use crate::metadata::PluginMetadata;
44use crate::params::{ParamKind, ParamSpec, ParamValue, ParamValues};
45use crate::plugin::{Plugin, PluginHandle};
46use crate::result::{
47    BenchmarkResult, Cell, CellStyle, Column, LogLine, ResultTable, RunStatus, Stat,
48};
49
50const SUMMARY: &str = "Pinned decode-rate floor: 3 fixed runs of one code prompt, \
51                       median server decode tok/s vs a committed threshold";
52pub const METADATA: PluginMetadata = PluginMetadata::metrale(SUMMARY);
53
54pub const DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
55    id: "decode-floor",
56    name: "Decode Floor Gate",
57    summary: SUMMARY,
58    detail: "Three timed streaming runs of one committed MinHeap-class code prompt, with every \
59             generation knob pinned (temperature 0, seed 0, max_tokens 1500, thinking off via \
60             a per-request reasoning_effort \"none\" — no serve flag needed). The metric is the \
61             MEDIAN server decode rate (usage.\"response_token/s\"), judged against the \
62             BENCH.toml floor under --pull-request-gate. Vacuity pins make the run \
63             INCONCLUSIVE rather than PASS when it measured nothing: every run must emit \
64             >=750 of the 1500-token budget (the calibrated instrument's natural stop is a \
65             deterministic 915), report the server rate, and show accept_len_mean >= 1.5 \
66             derived from usage.completion_tokens_details.accepted_prediction_tokens \
67             (requires the accept-stats instrumentation; a serve that is not speculating \
68             cannot pass this gate's floor honestly). REQUIRED since 2026-08-15; the \
69             floor in force is 22.7, from a 10-run set on the gate's own serve (2026-09-06).",
70    duration_hint: "~3–6 min",
71    expected_secs: 180,
72    updated: "2026-08-15",
73    // 2026-09-26: The only `gate = "decode-floor"` BENCH.toml entry is for
74    // unsloth/Qwen3.8-27B-NVFP4 (`kernels/gb10/qwen3.8-27b/BENCH.toml`); the
75    // driver measures whatever it is pointed at.
76    intended_for: Some(ModelExpectation {
77        families: &["qwen3.8-27b"],
78        note: "The decode floor is recorded for unsloth/Qwen3.8-27B-NVFP4 (10-run \
79               calibration on the gate's own serve, 2026-09-06). Other checkpoints run fine \
80               but have no committed floor to be judged against — a number with no baseline \
81               gates nothing.",
82    }),
83    // 2026-09-26: Under --pull-request-gate, `min_tok_s` is filled from the
84    // served variant's `server_decode_tok_s` bound (min minus noise) unless
85    // `--param` sets it, so a Measured run gives a pass or fail run verdict
86    // (`verdict_for`).
87    threshold_params: &[("min_tok_s", "server_decode_tok_s")],
88    needs_confirmation: false,
89    // 2026-09-26: The metric is a rate: a box that throttled during the run
90    // reports a floor miss that the code did not cause.
91    sensitivity: Sensitivity::Speed,
92    ctor: || Box::new(DecodeFloor::default()),
93};
94
95mod score;
96pub(crate) use score::{
97    Evaluation, MAX_TOKENS, MINHEAP_PROMPT, RUNS, RunObs, evaluate, instrument_metrics, verdict_for,
98};
99
100#[derive(Default)]
101pub struct DecodeFloor {
102    handle: Option<PluginHandle>,
103    timeout: Duration,
104    /// 2026-09-26: Verdict floor (tok/s); 0.0 gives an info verdict.
105    min_tok_s: f64,
106    samples: Vec<RunObs>,
107    started: Option<Instant>,
108    probed: bool,
109    /// 2026-09-26: GPU-rail power sampling: started after the probe (idle
110    /// baseline), one window per pinned run.
111    energy: EnergyMeter,
112}
113
114impl DecodeFloor {
115    fn request_body(model: &str) -> serde_json::Value {
116        json!({
117            "model": model,
118            "stream": true,
119            "temperature": 0.0,
120            "seed": 0,
121            "max_tokens": MAX_TOKENS,
122            "reasoning_effort": "none",
123            "messages": [{"role": "user", "content": MINHEAP_PROMPT}],
124        })
125    }
126
127    fn handle(&self) -> Result<&PluginHandle> {
128        self.handle.as_ref().context("benchmark was not loaded")
129    }
130
131    fn elapsed(&self) -> Duration {
132        self.started.map(|s| s.elapsed()).unwrap_or_default()
133    }
134
135    async fn one_run(&self) -> Result<http::ChatOutcome> {
136        let handle = self.handle()?;
137        let target = handle.target();
138        let body = Self::request_body(&target.model);
139        http::chat_stream(target, &body, self.timeout).await
140    }
141
142    fn table(&self) -> ResultTable {
143        let mut t = ResultTable::new(
144            "DECODE FLOOR",
145            vec![
146                Column::right("Run", 4),
147                Column::right("Out tok", 8),
148                Column::right("Decode tok/s (srv)", 18),
149                Column::right("Accepted", 9),
150                Column::right("E2E ms", 9),
151            ],
152        );
153        for (i, s) in self.samples.iter().enumerate() {
154            t.push(vec![
155                Cell::new((i + 1).to_string()),
156                Cell::new(s.completion_tokens.to_string()),
157                Cell::styled(
158                    s.server_tps
159                        .map(|v| format!("{v:.1}"))
160                        .unwrap_or_else(|| "—".into()),
161                    CellStyle::Accent,
162                ),
163                Cell::new(
164                    s.accepted_prediction_tokens
165                        .map(|v| v.to_string())
166                        .unwrap_or_else(|| "—".into()),
167                ),
168                Cell::new(format!("{:.0}", s.e2e_ms)),
169            ]);
170        }
171        t
172    }
173}
174
175impl Plugin for DecodeFloor {
176    fn metadata(&self) -> &'static PluginMetadata {
177        &METADATA
178    }
179
180    fn load(&mut self, handle: PluginHandle) -> impl Future<Output = Result<()>> + Send {
181        self.handle = Some(handle);
182        self.started = Some(Instant::now());
183        async { Ok(()) }
184    }
185}
186
187impl Benchmark for DecodeFloor {
188    fn descriptor(&self) -> &'static BenchmarkDescriptor {
189        &DESCRIPTOR
190    }
191
192    fn parameters(&self) -> Vec<ParamSpec> {
193        // 2026-09-26: The generation knobs are pins, not parameters (module
194        // docs). Only the transport timeout and the verdict floor are tunable.
195        vec![
196            ParamSpec::new(
197                "request_timeout_s",
198                "Request timeout",
199                "Seconds before a single request is abandoned. Transport-side only — it \
200                 cannot change the measured decode rate.",
201                ParamKind::Int { min: 30, max: 3600 },
202                ParamValue::Int(300),
203            ),
204            ParamSpec::new(
205                "min_tok_s",
206                "Decode floor",
207                "Run-verdict floor on the median server decode rate. 0 disables (a \
208                 standalone run reports an info verdict); under --pull-request-gate this \
209                 is auto-filled from the variant's BENCH.toml server_decode_tok_s `min` \
210                 bound. Vacuous runs stay INCONCLUSIVE regardless.",
211                ParamKind::Float {
212                    min: 0.0,
213                    max: 10_000.0,
214                },
215                // 2026-09-26: 0.0 is the documented off state, not an implicit
216                // bar (PCND).
217                ParamValue::Float(0.0),
218            ),
219        ]
220    }
221
222    fn configure(&mut self, values: &ParamValues) -> Result<()> {
223        let specs = self.parameters();
224        values.validate_against(&specs)?;
225        self.timeout = Duration::from_secs(values.usize("request_timeout_s")? as u64);
226        self.min_tok_s = values.float("min_tok_s")?;
227        self.samples.clear();
228        self.probed = false;
229        self.energy = EnergyMeter::default();
230        Ok(())
231    }
232
233    async fn next(&mut self) -> Result<BenchmarkResult> {
234        let handle = self.handle()?.clone();
235        handle.check_cancelled()?;
236        let total = RUNS as u64;
237
238        if !self.probed {
239            self.probed = true;
240            http::probe(handle.target(), Duration::from_secs(10))
241                .await
242                .context("endpoint probe failed — check the target URL and port")?;
243            // 2026-09-26: The probe answered and no run is in flight: the idle
244            // baseline is taken now.
245            for line in self.energy.start(handle.target()).await {
246                handle.log(line.level, line.text);
247            }
248            return Ok(BenchmarkResult::running("probe", self.elapsed())
249                .with_progress(0, total)
250                .log_line(LogLine::info(format!(
251                    "{} · MinHeap code prompt · max_tokens {MAX_TOKENS} · temp 0 · seed 0 · \
252                     reasoning_effort none · {RUNS} pinned runs",
253                    handle.target().base_url
254                ))));
255        }
256
257        if self.samples.len() < RUNS {
258            handle.status(format!("run {}/{RUNS}", self.samples.len() + 1));
259            let window_start = Instant::now();
260            let outcome = self.one_run().await?;
261            let window_end = Instant::now();
262            let mut obs = RunObs::from_outcome(&outcome);
263            obs.energy = self.energy.window(window_start, window_end);
264            let fmt_ms =
265                |v: Option<f64>| v.map(|v| format!("{v:.2}")).unwrap_or_else(|| "—".into());
266            let mut line = LogLine::info(format!(
267                "run {}/{RUNS}: {} tok · decode {} tok/s (server) · itl {} ms server / {} ms \
268                 client · accepted {} · E2E {:.0} ms",
269                self.samples.len() + 1,
270                obs.completion_tokens,
271                obs.server_tps
272                    .map(|v| format!("{v:.1}"))
273                    .unwrap_or_else(|| "—".into()),
274                fmt_ms(obs.server_tpot_ms),
275                fmt_ms(obs.client_tpot_ms),
276                obs.accepted_prediction_tokens
277                    .map(|v| v.to_string())
278                    .unwrap_or_else(|| "—".into()),
279                obs.e2e_ms,
280            ));
281            if let Some(e) = &obs.energy {
282                line.text
283                    .push_str(&format!(" · {}", e.one_line(self.energy.idle())));
284            }
285            self.samples.push(obs);
286            let done = self.samples.len() as u64;
287            handle.progress(done, total);
288            return Ok(BenchmarkResult::running("timed", self.elapsed())
289                .with_progress(done, total)
290                .with_table(self.table())
291                .log_line(line));
292        }
293
294        if self.samples.iter().all(|s| s.completion_tokens == 0) {
295            bail!("no run produced any output token — nothing to measure");
296        }
297
298        let sampler_cost_line = self.energy.stop().await;
299        let mut metrics = BTreeMap::new();
300        metrics.insert("runs".to_string(), self.samples.len() as f64);
301        instrument_metrics(&self.samples, self.energy.idle(), &mut metrics);
302        self.energy.metrics(&mut metrics);
303        let eval = evaluate(&self.samples);
304        let verdict = verdict_for(&eval, self.min_tok_s);
305        let summary = match eval {
306            Evaluation::Inconclusive(_) => Vec::new(),
307            Evaluation::Measured {
308                median_decode_tok_s,
309                min_output_tokens,
310                accept_len_mean,
311            } => {
312                metrics.insert("server_decode_tok_s".to_string(), median_decode_tok_s);
313                metrics.insert("output_tokens".to_string(), min_output_tokens as f64);
314                metrics.insert("accept_len_mean".to_string(), accept_len_mean);
315                vec![
316                    Stat::new(
317                        "Decode tok/s (server, median)",
318                        format!("{median_decode_tok_s:.1}"),
319                        "tok/s",
320                    )
321                    .with_style(CellStyle::Good),
322                    Stat::new("Accept len (mean)", format!("{accept_len_mean:.2}"), ""),
323                    Stat::new(
324                        "Output tok (min run)",
325                        format!("{min_output_tokens} / {MAX_TOKENS} cap"),
326                        "",
327                    ),
328                ]
329            }
330        };
331        let mut frame = BenchmarkResult {
332            status: RunStatus::Completed,
333            ..BenchmarkResult::running("done", self.elapsed())
334        }
335        .with_progress(total, total)
336        .with_summary(summary)
337        .with_table(self.table())
338        .with_metrics(metrics)
339        .with_verdict(verdict);
340        if let Some(line) = sampler_cost_line {
341            frame = frame.log_line(line);
342        }
343        Ok(frame)
344    }
345}
346
347#[cfg(test)]
348#[path = "decode_floor_tests.rs"]
349mod tests;
metrale_bench/benchmarks/decode_floor/mod.rs

metrale_bench/benchmarks/decode_floor/
mod.rs
