metrale_bench/benchmarks/
quick_speed.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Quick Speed Bench: warmup plus N timed streaming runs of one
4//! prompt, reporting the server's decode rate, TTFT and end-to-end latency. A
5//! measurement tool, not a gate: it stores no baseline, reads no threshold,
6//! and `gate::coverage::NOT_REQUIRED` lists it.
7//!
8//! Which tok/s to quote:
9//! * Decode tok/s (server), the headline: `usage."response_token/s"`, which
10//!   the server computes as `(completion_tokens - 1) / decode_time`
11//!   (`ir::Usage::decode_rate_tok_s`). It excludes prefill, client buffering
12//!   and the network.
13//! * E2E tok/s (client): `completion_tokens / wall`, including prefill and
14//!   transport.
15//!
16//! TPOT is derived from the server's decode rate (`1000 / server tok/s`), or
17//! omitted when the server reports none; no client-clock TPOT is emitted,
18//! because buffered reads and multi-token emission compress client-side
19//! arrival spacing.
20//!
21//! Every run, warmup included, sends the same prompt, so the timed TTFT is a
22//! warm figure. `ttft-warm-gate` and `ttft-cold-gate` control the prefix.
23//!
24//! Owner: bench, quick_speed.
25//! Invariants: each average counts only the runs that reported its value.
26
27use crate::hardware::Sensitivity;
28use std::collections::BTreeMap;
29use std::future::Future;
30use std::time::{Duration, Instant};
31
32use anyhow::{Context, Result, bail};
33use serde_json::json;
34
35use crate::benchmark::{Benchmark, BenchmarkDescriptor};
36use crate::benchmarks::stats::{self, PromptMode};
37use crate::http;
38use crate::metadata::PluginMetadata;
39use crate::params::{ParamKind, ParamSpec, ParamValue, ParamValues};
40use crate::plugin::{Plugin, PluginHandle};
41use crate::result::{
42    BenchmarkResult, Cell, CellStyle, Column, LogLine, ResultTable, RunStatus, Stat, Verdict,
43};
44
45const SUMMARY: &str = "Fast single-user speed probe: warmup + N timed runs, \
46                       server decode tok/s, TTFT and E2E";
47pub const METADATA: PluginMetadata = PluginMetadata::metrale(SUMMARY);
48
49pub const DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
50    id: "quick-speed-bench",
51    name: "Quick Speed Bench",
52    summary: SUMMARY,
53    detail: "One warmup plus N timed streaming runs of a single prompt. The headline is the \
54             SERVER-side decode-only rate ((completion_tokens − 1) / decode_time, from the \
55             response usage); the client E2E rate is shown separately and includes prefill and \
56             transport. TPOT is derived from the server's decode rate — the client-clock TPOT \
57             the old Python probe printed was a buffering artifact and is not reproduced. \
58             Because the warmup primes the Marconi prefix cache and per-session SSM snapshot, \
59             the timed TTFT is the warm INTRA-SESSION figure (~50–100 ms) — use the TTFT gates \
60             for controlled cold/warm measurements. A measurement tool: it gates nothing.",
61    duration_hint: "~1–3 min",
62    expected_secs: 150,
63    updated: "2026-08-15",
64    // 2026-09-26: A speed probe measures whatever it is pointed at; nothing
65    // here compares against a checkpoint-specific number.
66    intended_for: None,
67    threshold_params: &[],
68    needs_confirmation: false,
69    // 2026-09-26: Decode tok/s, TTFT and E2E: every headline is a speed number.
70    sensitivity: Sensitivity::Speed,
71    ctor: || Box::new(QuickSpeed::default()),
72};
73
74/// 2026-09-26: Committed prompt fixtures, keyed by the ISL they were cut for:
75/// real text rather than a looped filler corpus. Any other ISL falls back to
76/// [`stats::make_prompt`]'s synthesized filler.
77const FIXTURES: &[(usize, &str)] = &[
78    (
79        128,
80        include_str!("../../../../tests/fixtures/bench_prompt_128.txt"),
81    ),
82    (
83        512,
84        include_str!("../../../../tests/fixtures/bench_prompt_512.txt"),
85    ),
86    (
87        1024,
88        include_str!("../../../../tests/fixtures/bench_prompt_1024.txt"),
89    ),
90    (
91        4096,
92        include_str!("../../../../tests/fixtures/bench_prompt_4096.txt"),
93    ),
94];
95
96/// 2026-09-26: The counting instruction every prompt ends with (a fixture gets
97/// this constant; `make_prompt`'s `PromptMode::Count` appends the same text).
98/// Without it the model answers in a handful of tokens and the run measures
99/// scheduling overhead rather than decode.
100const COUNT_SUFFIX: &str = "Count from 1 upward, one number per line, until told to stop.";
101
102/// 2026-09-26: The prompt for a requested input length: a committed fixture
103/// when one exists for exactly that ISL, synthesized filler otherwise.
104pub(crate) fn prompt_for(isl: usize) -> String {
105    match FIXTURES.iter().find(|(n, _)| *n == isl) {
106        Some((_, text)) => format!("{}\n{COUNT_SUFFIX}", text.trim()),
107        None => stats::make_prompt(isl, PromptMode::Count, ""),
108    }
109}
110
111/// 2026-09-26: One timed run, reduced to the numbers this benchmark
112/// aggregates.
113#[derive(Clone, Debug, Default, PartialEq)]
114pub(crate) struct RunSample {
115    pub prompt_tokens: usize,
116    pub completion_tokens: usize,
117    pub e2e_ms: f64,
118    pub server_ttft_ms: Option<f64>,
119    pub server_tps: Option<f64>,
120}
121
122impl RunSample {
123    pub(crate) fn from_outcome(o: &http::ChatOutcome) -> Self {
124        Self {
125            prompt_tokens: o.prompt_tokens,
126            completion_tokens: o.completion_tokens,
127            e2e_ms: o.e2e_ms,
128            server_ttft_ms: o.server_ttft_ms,
129            server_tps: o.server_tps,
130        }
131    }
132
133    /// 2026-09-26: Client-observed rate, including prefill and transport.
134    pub(crate) fn client_e2e_tok_s(&self) -> Option<f64> {
135        (self.completion_tokens > 0 && self.e2e_ms.is_finite() && self.e2e_ms > 0.0)
136            .then(|| self.completion_tokens as f64 / (self.e2e_ms / 1000.0))
137    }
138
139    /// 2026-09-26: Per-token decode latency derived from the server's decode
140    /// rate: the only TPOT this benchmark reports.
141    pub(crate) fn server_tpot_ms(&self) -> Option<f64> {
142        self.server_tps
143            .filter(|t| t.is_finite() && *t > 0.0)
144            .map(|t| 1000.0 / t)
145    }
146}
147
148fn mean(values: impl IntoIterator<Item = f64>) -> Option<f64> {
149    let v: Vec<f64> = values.into_iter().filter(|x| x.is_finite()).collect();
150    (!v.is_empty()).then(|| v.iter().sum::<f64>() / v.len() as f64)
151}
152
153/// 2026-09-26: Cross-run averages. Each field averages only the runs that
154/// reported the value, so a run without server timings never contributes a
155/// zero.
156#[derive(Clone, Debug, Default, PartialEq)]
157pub(crate) struct Averages {
158    pub prompt_tokens: Option<f64>,
159    pub output_tokens: Option<f64>,
160    pub server_decode_tok_s: Option<f64>,
161    pub client_e2e_tok_s: Option<f64>,
162    pub server_ttft_ms: Option<f64>,
163    pub server_tpot_ms: Option<f64>,
164    pub e2e_ms: Option<f64>,
165}
166
167impl Averages {
168    pub(crate) fn of(samples: &[RunSample]) -> Self {
169        Self {
170            prompt_tokens: mean(samples.iter().map(|s| s.prompt_tokens as f64)),
171            output_tokens: mean(samples.iter().map(|s| s.completion_tokens as f64)),
172            server_decode_tok_s: mean(
173                samples
174                    .iter()
175                    .filter_map(|s| s.server_tps)
176                    .filter(|v| v.is_finite() && *v > 0.0),
177            ),
178            client_e2e_tok_s: mean(samples.iter().filter_map(RunSample::client_e2e_tok_s)),
179            server_ttft_ms: mean(
180                samples
181                    .iter()
182                    .filter_map(|s| s.server_ttft_ms)
183                    .filter(|v| v.is_finite() && *v >= 0.0),
184            ),
185            server_tpot_ms: mean(samples.iter().filter_map(RunSample::server_tpot_ms)),
186            e2e_ms: mean(
187                samples
188                    .iter()
189                    .map(|s| s.e2e_ms)
190                    .filter(|v| v.is_finite() && *v >= 0.0),
191            ),
192        }
193    }
194}
195
196fn fmt(v: Option<f64>, digits: usize) -> String {
197    v.map(|x| format!("{x:.digits$}"))
198        .unwrap_or_else(|| "—".into())
199}
200
201#[derive(Default)]
202pub struct QuickSpeed {
203    handle: Option<PluginHandle>,
204    isl: usize,
205    osl: usize,
206    runs: usize,
207    warmup: usize,
208    timeout: Duration,
209    warmups_done: usize,
210    samples: Vec<RunSample>,
211    started: Option<Instant>,
212    probed: bool,
213}
214
215impl QuickSpeed {
216    fn request_body(&self, model: &str) -> serde_json::Value {
217        json!({
218            "model": model,
219            "stream": true,
220            "max_tokens": self.osl,
221            "temperature": 0.0,
222            "messages": [{"role": "user", "content": prompt_for(self.isl)}],
223        })
224    }
225
226    fn handle(&self) -> Result<&PluginHandle> {
227        self.handle.as_ref().context("benchmark was not loaded")
228    }
229
230    fn elapsed(&self) -> Duration {
231        self.started.map(|s| s.elapsed()).unwrap_or_default()
232    }
233
234    async fn one_run(&self) -> Result<http::ChatOutcome> {
235        let handle = self.handle()?;
236        let target = handle.target();
237        let body = self.request_body(&target.model);
238        http::chat_stream(target, &body, self.timeout).await
239    }
240
241    fn table(&self) -> ResultTable {
242        let mut t = ResultTable::new(
243            "QUICK SPEED",
244            vec![
245                Column::right("Run", 4),
246                Column::right("Out tok", 8),
247                Column::right("Decode tok/s (srv)", 18),
248                Column::right("TTFT ms (srv)", 13),
249                Column::right("E2E ms", 9),
250            ],
251        );
252        for (i, s) in self.samples.iter().enumerate() {
253            t.push(vec![
254                Cell::new((i + 1).to_string()),
255                Cell::new(s.completion_tokens.to_string()),
256                Cell::styled(fmt(s.server_tps, 1), CellStyle::Accent),
257                Cell::new(fmt(s.server_ttft_ms, 0)),
258                Cell::new(format!("{:.0}", s.e2e_ms)),
259            ]);
260        }
261        t
262    }
263
264    fn summary(&self, avg: &Averages) -> Vec<Stat> {
265        vec![
266            // 2026-09-26: The headline; its label names the server rate so the
267            // two tok/s figures cannot be confused.
268            Stat::new(
269                "Decode tok/s (server)",
270                fmt(avg.server_decode_tok_s, 1),
271                "tok/s",
272            )
273            .with_style(CellStyle::Good),
274            Stat::new(
275                "E2E tok/s (client, incl. prefill)",
276                fmt(avg.client_e2e_tok_s, 1),
277                "tok/s",
278            )
279            .with_style(CellStyle::Dim),
280            Stat::new("TTFT (server prefill)", fmt(avg.server_ttft_ms, 1), "ms")
281                .with_style(CellStyle::Accent),
282            Stat::new("TPOT (server decode)", fmt(avg.server_tpot_ms, 2), "ms"),
283            Stat::new(
284                "Output tok",
285                format!("{} / {} cap", fmt(avg.output_tokens, 0), self.osl),
286                "",
287            ),
288        ]
289    }
290
291    fn metrics(&self, avg: &Averages) -> BTreeMap<String, f64> {
292        let mut m = BTreeMap::new();
293        m.insert("runs".to_string(), self.samples.len() as f64);
294        let pairs = [
295            ("prompt_tokens", avg.prompt_tokens),
296            ("output_tokens", avg.output_tokens),
297            ("server_decode_tok_s", avg.server_decode_tok_s),
298            ("client_e2e_tok_s", avg.client_e2e_tok_s),
299            ("server_ttft_ms", avg.server_ttft_ms),
300            ("server_tpot_ms", avg.server_tpot_ms),
301            ("e2e_ms", avg.e2e_ms),
302        ];
303        for (k, v) in pairs {
304            if let Some(v) = v {
305                m.insert(k.to_string(), v);
306            }
307        }
308        m
309    }
310}
311
312impl Plugin for QuickSpeed {
313    fn metadata(&self) -> &'static PluginMetadata {
314        &METADATA
315    }
316
317    fn load(&mut self, handle: PluginHandle) -> impl Future<Output = Result<()>> + Send {
318        self.handle = Some(handle);
319        self.started = Some(Instant::now());
320        async { Ok(()) }
321    }
322}
323
324impl Benchmark for QuickSpeed {
325    fn descriptor(&self) -> &'static BenchmarkDescriptor {
326        &DESCRIPTOR
327    }
328
329    fn parameters(&self) -> Vec<ParamSpec> {
330        vec![
331            ParamSpec::new(
332                "isl",
333                "Input length",
334                "Prompt size in tokens. 128 / 512 / 1024 / 4096 load a committed text fixture; \
335                 any other size synthesizes a filler prompt.",
336                ParamKind::Int {
337                    min: 16,
338                    max: 131_072,
339                },
340                ParamValue::Int(60),
341            ),
342            ParamSpec::new(
343                "osl",
344                "Output tokens",
345                "A CEILING, not a target — the model may hit EOS earlier, and a 49-token reply \
346                 against a 128 cap is the model stopping, not a defect. The counting suffix \
347                 pushes toward the cap but does not guarantee it.",
348                ParamKind::Int { min: 2, max: 8192 },
349                ParamValue::Int(128),
350            ),
351            ParamSpec::new(
352                "runs",
353                "Timed runs",
354                "Measured requests; the report averages across them.",
355                ParamKind::Int { min: 1, max: 100 },
356                ParamValue::Int(5),
357            ),
358            ParamSpec::new(
359                "warmup",
360                "Warm-up runs",
361                "Unmeasured priming requests with the SAME prompt. They populate the Marconi \
362                 prefix cache and the per-session SSM snapshot, so the timed TTFT is the warm \
363                 intra-session figure — use the TTFT gates to measure cold prefill.",
364                ParamKind::Int { min: 0, max: 10 },
365                ParamValue::Int(1),
366            ),
367            ParamSpec::new(
368                "request_timeout_s",
369                "Request timeout",
370                "Seconds before a single request is abandoned.",
371                ParamKind::Int { min: 10, max: 3600 },
372                ParamValue::Int(300),
373            ),
374        ]
375    }
376
377    fn configure(&mut self, values: &ParamValues) -> Result<()> {
378        let specs = self.parameters();
379        values.validate_against(&specs)?;
380        self.isl = values.usize("isl")?;
381        self.osl = values.usize("osl")?;
382        self.runs = values.usize("runs")?;
383        self.warmup = values.usize("warmup")?;
384        self.timeout = Duration::from_secs(values.usize("request_timeout_s")? as u64);
385        self.warmups_done = 0;
386        self.samples.clear();
387        self.probed = false;
388        Ok(())
389    }
390
391    async fn next(&mut self) -> Result<BenchmarkResult> {
392        let handle = self.handle()?.clone();
393        handle.check_cancelled()?;
394        let total = (self.warmup + self.runs) as u64;
395        let done = (self.warmups_done + self.samples.len()) as u64;
396
397        if !self.probed {
398            self.probed = true;
399            http::probe(handle.target(), Duration::from_secs(10))
400                .await
401                .context("endpoint probe failed — check the target URL and port")?;
402            return Ok(BenchmarkResult::running("probe", self.elapsed())
403                .with_progress(0, total)
404                .log_line(LogLine::info(format!(
405                    "{} · isl {} · osl {} (cap) · {} warmup + {} timed",
406                    handle.target().base_url,
407                    self.isl,
408                    self.osl,
409                    self.warmup,
410                    self.runs
411                ))));
412        }
413
414        if self.warmups_done < self.warmup {
415            handle.status(format!("warmup {}/{}", self.warmups_done + 1, self.warmup));
416            let outcome = self.one_run().await?;
417            self.warmups_done += 1;
418            return Ok(BenchmarkResult::running("warmup", self.elapsed())
419                .with_progress(done + 1, total)
420                .log_line(LogLine::info(format!(
421                    "warmup {}/{}: {} prompt tokens (prefix cache + SSM snapshot primed)",
422                    self.warmups_done, self.warmup, outcome.prompt_tokens
423                ))));
424        }
425
426        if self.samples.len() < self.runs {
427            handle.status(format!("run {}/{}", self.samples.len() + 1, self.runs));
428            let outcome = self.one_run().await?;
429            let sample = RunSample::from_outcome(&outcome);
430            let line = LogLine::info(format!(
431                "run {}/{}: {} tok · decode {} tok/s (server) · TTFT {} ms · E2E {:.0} ms",
432                self.samples.len() + 1,
433                self.runs,
434                sample.completion_tokens,
435                fmt(sample.server_tps, 1),
436                fmt(sample.server_ttft_ms, 0),
437                sample.e2e_ms,
438            ));
439            self.samples.push(sample);
440            handle.progress(done + 1, total);
441            return Ok(BenchmarkResult::running("timed", self.elapsed())
442                .with_progress(done + 1, total)
443                .with_table(self.table())
444                .log_line(line));
445        }
446
447        if self.samples.iter().all(|s| s.completion_tokens == 0) {
448            bail!("no run produced any output token — nothing to measure");
449        }
450        let avg = Averages::of(&self.samples);
451        let verdict = Verdict::info(format!(
452            "measured {} run(s): decode {} tok/s (server), TTFT {} ms — a measurement, \
453             not a gate",
454            self.samples.len(),
455            fmt(avg.server_decode_tok_s, 1),
456            fmt(avg.server_ttft_ms, 1),
457        ));
458        Ok(BenchmarkResult {
459            status: RunStatus::Completed,
460            ..BenchmarkResult::running("done", self.elapsed())
461        }
462        .with_progress(total, total)
463        .with_summary(self.summary(&avg))
464        .with_table(self.table())
465        .with_metrics(self.metrics(&avg))
466        .with_verdict(verdict))
467    }
468}
469
470#[cfg(test)]
471#[path = "quick_speed_tests.rs"]
472mod tests;
metrale_bench/benchmarks/quick_speed.rs

metrale_bench/benchmarks/
quick_speed.rs
