metrale_bench/benchmarks/ttft/
descriptors.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The descriptors and metadata of the TTFT gates: warm (cached
4//! prefix) and cold (uncached prefill), over synthetic prompts of several
5//! lengths or (2026-09-27) the committed 32k-token prompt, one high-ISL pair
6//! per subject. Each compares a run with a baseline the gate stores for the
7//! same host and model.
8//!
9//! Owner: bench, ttft.
10//! Invariants: none beyond the types.
11
12use super::{Mode, TtftGate};
13use crate::benchmark::{BenchmarkDescriptor, ModelExpectation};
14use crate::hardware::Sensitivity;
15use crate::metadata::PluginMetadata;
16
17const WARM_SUMMARY: &str = "Cached-prefix TTFT vs the stored same-box baseline";
18const COLD_SUMMARY: &str = "Uncached prefill TTFT vs the stored same-box baseline";
19pub const WARM_METADATA: PluginMetadata = PluginMetadata::metrale(WARM_SUMMARY);
20pub const COLD_METADATA: PluginMetadata = PluginMetadata::metrale(COLD_SUMMARY);
21
22pub const WARM_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
23    id: "ttft-warm-gate",
24    name: "Warm TTFT Regression Gate",
25    summary: WARM_SUMMARY,
26    detail: "Measures time-to-first-token on the WARM path: each sample repeats a bit-identical \
27             prompt so the prefix cache hits. Gates at median ≤3% and p90 ≤5% against a baseline \
28             recorded on this box — the guard that catches an optimization silently falling back \
29             to a slow path while the correctness gates stay green.",
30    duration_hint: "~3–6 min",
31    expected_secs: 160,
32    updated: "2026-07-31",
33    needs_confirmation: false,
34    // 2026-09-26: The gate compares with a baseline it stores itself, keyed by
35    // model and checked for the same host, so it applies to any checkpoint.
36    intended_for: None,
37    threshold_params: &[],
38    // 2026-09-26: Speed: the verdict compares TTFT latencies, which a thermal
39    // throttle during the run moves.
40    sensitivity: Sensitivity::Speed,
41    ctor: || Box::new(TtftGate::new(Mode::Warm)),
42};
43
44pub const COLD_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
45    id: "ttft-cold-gate",
46    name: "Cold TTFT Regression Gate",
47    summary: COLD_SUMMARY,
48    detail: "Measures time-to-first-token with the prefix cache guaranteed to MISS: every sample \
49             carries a unique prefix_tag, so each request pays a full prefill. This is the prefill path \
50             on its own, with the cache's contribution removed — the warm gate cannot see a \
51             prefill regression that caching is hiding.",
52    duration_hint: "~3–6 min",
53    expected_secs: 120,
54    updated: "2026-07-31",
55    needs_confirmation: false,
56    // 2026-09-26: The gate compares with a baseline it stores itself, keyed by
57    // model and checked for the same host, so it applies to any checkpoint.
58    intended_for: None,
59    threshold_params: &[],
60    // 2026-09-26: Speed, for the same reason as the warm gate.
61    sensitivity: Sensitivity::Speed,
62    ctor: || Box::new(TtftGate::new(Mode::Cold)),
63};
64
65const HIGH_ISL_COLD_SUMMARY: &str = "Uncached 32k-token prefill TTFT on the dense flagship";
66const HIGH_ISL_WARM_SUMMARY: &str = "Cached 32k-token prefix TTFT on the dense flagship";
67const HIGH_ISL_COLD_MOE_SUMMARY: &str = "Uncached 32k-token prefill TTFT on the 35B MoE";
68const HIGH_ISL_WARM_MOE_SUMMARY: &str = "Cached 32k-token prefix TTFT on the 35B MoE";
69const HIGH_ISL_COLD_METADATA: PluginMetadata = PluginMetadata::metrale(HIGH_ISL_COLD_SUMMARY);
70const HIGH_ISL_WARM_METADATA: PluginMetadata = PluginMetadata::metrale(HIGH_ISL_WARM_SUMMARY);
71const HIGH_ISL_COLD_MOE_METADATA: PluginMetadata =
72    PluginMetadata::metrale(HIGH_ISL_COLD_MOE_SUMMARY);
73const HIGH_ISL_WARM_MOE_METADATA: PluginMetadata =
74    PluginMetadata::metrale(HIGH_ISL_WARM_MOE_SUMMARY);
75
76/// 2026-09-27: The high-ISL gates' checkpoint families: one default subject per
77/// gate id, as with `concurrency-sweep` and `concurrency-sweep-moe`.
78const DENSE: ModelExpectation = ModelExpectation {
79    families: &["qwen3.8-27b"],
80    note: "The high-ISL pair without the -moe suffix is defined on the dense Qwen3.8-27B \
81           flagship; the -moe pair measures the Qwen3.6-35B-A3B MoE.",
82};
83const MOE: ModelExpectation = ModelExpectation {
84    families: &["qwen3.6-35b-a3b"],
85    note: "The -moe high-ISL pair is defined on the Qwen3.6-35B-A3B MoE; the pair without \
86           the suffix measures the dense Qwen3.8-27B flagship.",
87};
88
89/// 2026-09-27: `expected_secs` below are one-shot runs (owner, 2026-09-27) at
90/// the 2026-09-27 measurements: one 32k-token prefill per cold run (MoE 14 s,
91/// dense 45 s), plus a priming prefill before the warm one.
92pub const HIGH_ISL_COLD_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
93    id: "high-isl-ttft-cold",
94    name: "High-ISL Cold TTFT Gate",
95    summary: HIGH_ISL_COLD_SUMMARY,
96    detail: "Time-to-first-token on a committed 32k-token prompt (Moby-Dick, see \
97             ttft/prompts/NOTICE.md), one-shot: an unmeasured short warm-up request, then \
98             one request with a unique tag at the start, so it pays the whole prefill. The \
99             server's usage.prompt_tokens is checked: a missing count or one below \
100             min_prompt_tokens makes the run invalid. Compared with a same-box baseline like \
101             ttft-cold-gate; the BENCH.toml ceiling is vLLM's TTFT on the same prompt and box.",
102    duration_hint: "~1–2 min",
103    expected_secs: 70,
104    updated: "2026-09-27",
105    needs_confirmation: false,
106    intended_for: Some(DENSE),
107    threshold_params: &[],
108    // 2026-09-27: Speed, as for the synthetic TTFT gates.
109    sensitivity: Sensitivity::Speed,
110    ctor: || {
111        Box::new(TtftGate::high_isl(
112            Mode::Cold,
113            &HIGH_ISL_COLD_DESCRIPTOR,
114            &HIGH_ISL_COLD_METADATA,
115        ))
116    },
117};
118
119pub const HIGH_ISL_WARM_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
120    id: "high-isl-ttft-warm",
121    name: "High-ISL Warm TTFT Gate",
122    summary: HIGH_ISL_WARM_SUMMARY,
123    detail: "Time-to-first-token on the committed 32k-token prompt with one fixed tag, \
124             one-shot: an unmeasured short warm-up request, then the prompt is primed and \
125             re-sent byte for byte, so the prefix cache holds the whole prompt. The server's \
126             usage.prompt_tokens is checked, as in high-isl-ttft-cold.",
127    duration_hint: "~1–2 min",
128    expected_secs: 80,
129    updated: "2026-09-27",
130    needs_confirmation: false,
131    intended_for: Some(DENSE),
132    threshold_params: &[],
133    sensitivity: Sensitivity::Speed,
134    ctor: || {
135        Box::new(TtftGate::high_isl(
136            Mode::Warm,
137            &HIGH_ISL_WARM_DESCRIPTOR,
138            &HIGH_ISL_WARM_METADATA,
139        ))
140    },
141};
142
143pub const HIGH_ISL_COLD_MOE_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
144    id: "high-isl-ttft-cold-moe",
145    name: "High-ISL Cold TTFT Gate (MoE)",
146    summary: HIGH_ISL_COLD_MOE_SUMMARY,
147    detail: "high-isl-ttft-cold on the Qwen3.6-35B-A3B MoE: the same prompt and the same \
148             checks, with its own id so its baseline and bounds are read against its own \
149             history only.",
150    duration_hint: "~1 min",
151    expected_secs: 30,
152    updated: "2026-09-27",
153    needs_confirmation: false,
154    intended_for: Some(MOE),
155    threshold_params: &[],
156    sensitivity: Sensitivity::Speed,
157    ctor: || {
158        Box::new(TtftGate::high_isl(
159            Mode::Cold,
160            &HIGH_ISL_COLD_MOE_DESCRIPTOR,
161            &HIGH_ISL_COLD_MOE_METADATA,
162        ))
163    },
164};
165
166pub const HIGH_ISL_WARM_MOE_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
167    id: "high-isl-ttft-warm-moe",
168    name: "High-ISL Warm TTFT Gate (MoE)",
169    summary: HIGH_ISL_WARM_MOE_SUMMARY,
170    detail: "high-isl-ttft-warm on the Qwen3.6-35B-A3B MoE: the same prompt and the same \
171             checks, with its own id so its baseline and bounds are read against its own \
172             history only.",
173    duration_hint: "~1 min",
174    expected_secs: 30,
175    updated: "2026-09-27",
176    needs_confirmation: false,
177    intended_for: Some(MOE),
178    threshold_params: &[],
179    sensitivity: Sensitivity::Speed,
180    ctor: || {
181        Box::new(TtftGate::high_isl(
182            Mode::Warm,
183            &HIGH_ISL_WARM_MOE_DESCRIPTOR,
184            &HIGH_ISL_WARM_MOE_METADATA,
185        ))
186    },
187};
metrale_bench/benchmarks/ttft/descriptors.rs

metrale_bench/benchmarks/ttft/
descriptors.rs
