metrale_bench/benchmarks/
concurrency_descriptors.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The concurrency sweep's registry entries: the three gate
4//! descriptors that run the `ConcurrencySweep` driver, and the rung table
5//! their floor params and metric keys are derived from.
6//!
7//! Owner: bench (concurrency).
8//! Invariants: every `GATE_THRESHOLD_PARAMS` pair comes from `RUNGS` or
9//! `PEAK_FLOOR`.
10
11use super::{BenchmarkDescriptor, ConcurrencySweep, PluginMetadata, Sensitivity};
12
13const SUMMARY: &str = "Latency/throughput curve across concurrency 1 → 128";
14pub const METADATA: PluginMetadata = PluginMetadata::metrale(SUMMARY);
15
16pub const DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
17    id: "concurrency-sweep",
18    name: "Concurrency Sweep",
19    summary: SUMMARY,
20    detail: "Fires N concurrent streaming requests per (input-length × concurrency) cell and \
21             reports client TTFT, TPOT and end-to-end latency as p50/p90/p99, plus the batch's \
22             aggregate output throughput. This is the curve the GB10 concurrency campaign is \
23             measured on — C=1 is where Metrale Engine leads, C=32 is where time-to-answer starts \
24             inverting in Metrale Engine's favour, and C=128 is the widest rung the published ladder \
25             quotes. Requests pin temperature 0.0 / \
26             seed 0 and send reasoning_effort \"none\" so the ladder measures decode, not \
27             thinking. A cell that delivers under 80% of its total output budget, or whose \
28             median request delivers under 80% of its own, is flagged vacuous and its tok/s \
29             marked non-comparable (`concurrency_vacuity.rs`). REQUIRED gate since \
30             2026-08-15: under --pull-request-gate the run serves the calibrated instrument \
31             (C=1..128, isl 512, osl 320 via the variant's param_overrides) and \
32             self-verdicts against gate-filled per-rung floors; a sweep with any vacuous \
33             cell or request error never passes, whatever the floors say.",
34    duration_hint: "~25–90 min",
35    expected_secs: 1560,
36    updated: "2026-08-29",
37    needs_confirmation: false,
38    // 2026-09-26: Any served model is a valid subject; no threshold here is
39    // tied to a checkpoint.
40    intended_for: None,
41    // 2026-09-26: Under --pull-request-gate each floor is filled from the
42    // variant's BENCH.toml `min` bound minus its noise (the server's
43    // `apply_threshold_params`). The gated instrument (rungs, ISL, OSL,
44    // fixture) comes from the entry's `param_overrides`, not from these schema
45    // defaults.
46    threshold_params: GATE_THRESHOLD_PARAMS,
47    // 2026-09-26: A thermal event mid-sweep slows the later rungs only, which
48    // reads as a change in the curve's shape.
49    sensitivity: Sensitivity::Speed,
50    ctor: || Box::new(ConcurrencySweep::default()),
51};
52
53/// 2026-09-26: The concurrency gate served with the DFlash2 drafter armed. A
54/// separate gate id rather than a BENCH.toml variant: a required gate counts
55/// only records of its declared default checkpoint
56/// (`gate::check::record_is_required_subject`). It shares this driver and its
57/// floor params with `concurrency-sweep`; its BENCH.toml entry names a
58/// DFlash2 recipe and its own instrument and floors.
59pub const DFLASH2_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
60    id: "concurrency-sweep-dflash2",
61    name: "Concurrency Sweep (DFlash2)",
62    summary: DFLASH2_SUMMARY,
63    detail: "The concurrency ladder with the DFlash2 block-diffusion drafter armed \
64             (`--dflash --draft-model incoai/Qwen3.8-27B-DFlash2 --dflash-gamma 8`), pinned by \
65             the variant's serve_overrides. Same fixture, same rungs and same vacuity rule as \
66             `concurrency-sweep`; the only difference is that the served engine speculates, \
67             which is exactly the path no other required gate exercises. Expect the two curves \
68             to converge at the wide rungs: DFlash2's verify batches are bounded, so above the \
69             point where speculation self-limits this measures the base engine and says so \
70             rather than pretending otherwise.",
71    duration_hint: "~25–90 min",
72    expected_secs: 300,
73    updated: "2026-08-29",
74    needs_confirmation: false,
75    intended_for: Some(crate::benchmark::ModelExpectation {
76        families: &["qwen3.8-27b"],
77        note: "DFlash2 drafters are trained against one target's hidden states \
78               (incoai/Qwen3.8-27B-DFlash2 consumes target layers [5,19,33,47,61] of \
79               Qwen3.8-27B). Pointing this gate at another checkpoint measures a mismatched \
80               drafter, which is slow rather than wrong and therefore easy to misread.",
81    }),
82    threshold_params: GATE_THRESHOLD_PARAMS,
83    sensitivity: Sensitivity::Speed,
84    ctor: || Box::new(ConcurrencySweep::default()),
85};
86
87const DFLASH2_SUMMARY: &str = "Latency/throughput curve across concurrency 1 → 128, DFlash2 armed";
88
89/// 2026-09-26: The concurrency gate on the Qwen3.6-35B-A3B MoE. A separate gate
90/// id, for the reason the DFlash2 one is: a required gate scores only its
91/// declared default checkpoint per box class
92/// (`gate::check::record_is_required_subject`), and `gate::bench::baseline_for`
93/// refuses two defaults on one box class. Its BENCH.toml entry pins the
94/// published ladder's instrument (ISL 128, OSL 1024, the `essay` fixture,
95/// C=1..16), the one the repository's vLLM baseline for this checkpoint
96/// (`bench/baselines/qwen36-35b-a3b/published.json`) was measured on.
97pub const MOE_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
98    id: "concurrency-sweep-moe",
99    name: "Concurrency Sweep (MoE)",
100    summary: MOE_SUMMARY,
101    detail: "The concurrency ladder on the Qwen3.6-35B-A3B MoE flagship, pinned by the \
102             variant's param_overrides to the PUBLISHED instrument the vLLM one-shot for \
103             this checkpoint was measured on: ISL 128 / OSL 1024, the ladder38 essay \
104             request byte for byte, C=1..16. Same driver, same rungs-and-floors shape and \
105             same vacuity rule as `concurrency-sweep`; the MoE decode path takes the \
106             grouped-GEMM expert arm above the width gate that the dense ladder never \
107             reaches, which is why a dense record cannot speak for it. Its numbers are \
108             NOT comparable to the dense gates' (a different checkpoint on a different \
109             instrument, ~4x apart) and each is read against its own history only.",
110    duration_hint: "~5–15 min",
111    expected_secs: 600,
112    updated: "2026-09-20",
113    needs_confirmation: false,
114    intended_for: Some(crate::benchmark::ModelExpectation {
115        families: &["qwen3.6-35b-a3b"],
116        note: "The MoE ladder is defined on the Qwen3.6-35B-A3B family (the FP8 flagship \
117               is its declared subject). Pointing it at the dense 27B measures the dense \
118               FFN path under the MoE's instrument — a number with no history and no floor.",
119    }),
120    threshold_params: GATE_THRESHOLD_PARAMS,
121    sensitivity: Sensitivity::Speed,
122    ctor: || Box::new(ConcurrencySweep::default()),
123};
124
125const MOE_SUMMARY: &str =
126    "Latency/throughput curve across concurrency 1 → 16 on the 35B MoE, published instrument";
127
128/// 2026-09-26: The gated rungs, `(C, floor param, metric key, label)`. The
129/// descriptors' `threshold_params`, the floor `ParamSpec`s and `configure`'s
130/// `Floors::per_c` are all derived from this table. A rung with no bound in
131/// the variant's BENCH.toml keeps its 0.0 default and gates nothing.
132pub(super) const RUNGS: [(usize, &str, &str, &str); 8] = [
133    (1, "min_c1", "c1_aggregate_tok_s", "C=1 aggregate floor"),
134    (2, "min_c2", "c2_aggregate_tok_s", "C=2 aggregate floor"),
135    (4, "min_c4", "c4_aggregate_tok_s", "C=4 aggregate floor"),
136    (8, "min_c8", "c8_aggregate_tok_s", "C=8 aggregate floor"),
137    (16, "min_c16", "c16_aggregate_tok_s", "C=16 aggregate floor"),
138    (32, "min_c32", "c32_aggregate_tok_s", "C=32 aggregate floor"),
139    (64, "min_c64", "c64_aggregate_tok_s", "C=64 aggregate floor"),
140    (
141        128,
142        "min_c128",
143        "c128_aggregate_tok_s",
144        "C=128 aggregate floor",
145    ),
146];
147
148/// 2026-09-26: Not a rung: it bounds `peak_aggregate_tok_s`, whichever C
149/// produced it.
150pub(super) const PEAK_FLOOR: (&str, &str, &str) =
151    ("min_peak", "peak_aggregate_tok_s", "Peak aggregate floor");
152
153/// 2026-09-26: Each floor param paired with the metric its BENCH.toml bound is
154/// written on, built from [`RUNGS`] and [`PEAK_FLOOR`] so every pair has a
155/// matching `ParamSpec`. The server's `apply_threshold_params` fails on a pair
156/// without one once the baseline bounds that metric.
157const GATE_THRESHOLD_PARAMS: &[(&str, &str)] = &gate_threshold_params();
158
159const fn gate_threshold_params() -> [(&'static str, &'static str); RUNGS.len() + 1] {
160    let mut out = [("", ""); RUNGS.len() + 1];
161    let mut i = 0;
162    while i < RUNGS.len() {
163        out[i] = (RUNGS[i].1, RUNGS[i].2);
164        i += 1;
165    }
166    out[RUNGS.len()] = (PEAK_FLOOR.0, PEAK_FLOOR.1);
167    out
168}
metrale_bench/benchmarks/concurrency_descriptors.rs

metrale_bench/benchmarks/
concurrency_descriptors.rs
