metrale_bench/http/
gaps.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Arrival-gap accumulator: the within-response jitter instrument.
4//!
5//! An arrival is one socket read that carried at least one token delta
6//! (`chat_stream_inner` in `http.rs`); a gap is the time between two
7//! consecutive arrivals of one response. Gaps are taken per read, not per
8//! token: several deltas that arrive in one read are one arrival.
9//!
10//! Count, mean and variance are kept online (Welford in `push`, Chan's
11//! update in `merge`) with a running max. Percentiles need the samples, so
12//! gaps are retained as `f32` up to [`RETAIN_CAP`] per response; past that
13//! the online statistics keep counting and [`GapStats::retained`] says how
14//! many gaps the percentiles cover. A cell pools its requests with
15//! [`GapSample::merge`]. `stability` and the coefficient of variation are
16//! derived from the primitives by [`GapStats`]; for both, lower is better.
17//!
18//! Owner: bench (HTTP client).
19//! Invariants:
20//! - A `GapSample` never retains more gaps than it has counted.
21//! - [`GapSample::stats`] is `None` exactly when no gap was recorded.
22
23use std::collections::BTreeMap;
24
25use crate::benchmarks::stats;
26
27/// 2026-09-26: Gaps retained per response for the percentiles. Equal to the
28/// concurrency driver's largest `osl`, which is its `max_tokens`, so no
29/// concurrency response is truncated; a longer response keeps counting
30/// online and reports the truncation through [`GapStats::retained`].
31pub const RETAIN_CAP: usize = 8192;
32
33/// 2026-09-26: Online accumulator of one response's arrival gaps (or a pool of them).
34#[derive(Clone, Debug, Default, PartialEq)]
35pub struct GapSample {
36    count: u64,
37    mean: f64,
38    m2: f64,
39    max: f64,
40    retained: Vec<f32>,
41}
42
43impl GapSample {
44    /// 2026-09-26: Record one gap, in milliseconds.
45    pub fn push(&mut self, gap_ms: f64) {
46        self.count += 1;
47        let delta = gap_ms - self.mean;
48        self.mean += delta / self.count as f64;
49        self.m2 += delta * (gap_ms - self.mean);
50        if gap_ms > self.max {
51            self.max = gap_ms;
52        }
53        if self.retained.len() < RETAIN_CAP {
54            self.retained.push(gap_ms as f32);
55        }
56    }
57
58    /// 2026-09-26: Pool another sample into this one (Chan's parallel update
59    /// for the moments; the retained gaps concatenate, so a pool of `k`
60    /// responses holds at most `k × RETAIN_CAP` gaps).
61    pub fn merge(&mut self, other: &GapSample) {
62        if other.count == 0 {
63            return;
64        }
65        let na = self.count as f64;
66        let nb = other.count as f64;
67        let n = na + nb;
68        let delta = other.mean - self.mean;
69        self.mean += delta * nb / n;
70        self.m2 += other.m2 + delta * delta * na * nb / n;
71        self.count += other.count;
72        self.max = self.max.max(other.max);
73        self.retained.extend_from_slice(&other.retained);
74    }
75
76    pub fn count(&self) -> u64 {
77        self.count
78    }
79
80    /// 2026-09-26: The distribution summary, or `None` when no gap was
81    /// recorded (fewer than two arrivals): a zero would read as "perfectly
82    /// smooth".
83    pub fn stats(&self) -> Option<GapStats> {
84        if self.count == 0 {
85            return None;
86        }
87        let retained: Vec<f64> = self.retained.iter().map(|g| f64::from(*g)).collect();
88        let pct = |p| stats::percentile(&retained, p).unwrap_or(f64::NAN);
89        Some(GapStats {
90            count: self.count,
91            retained: self.retained.len(),
92            mean_ms: self.mean,
93            // 2026-09-26: Population deviation: the gaps are the whole
94            // population of this response, not a sample of a larger one.
95            stddev_ms: (self.m2 / self.count as f64).sqrt(),
96            max_ms: self.max,
97            p50_ms: pct(50),
98            p90_ms: pct(90),
99            p99_ms: pct(99),
100        })
101    }
102}
103
104/// 2026-09-26: One response's (or one pool's) arrival-gap distribution, in ms.
105#[derive(Clone, Copy, Debug, Default, PartialEq)]
106pub struct GapStats {
107    /// 2026-09-26: Gaps observed.
108    pub count: u64,
109    /// 2026-09-26: Gaps the percentiles were computed over (`< count` only
110    /// when a response ran past [`RETAIN_CAP`]).
111    pub retained: usize,
112    pub mean_ms: f64,
113    pub stddev_ms: f64,
114    /// 2026-09-26: A stall is a tail event; the mean hides it and this does
115    /// not.
116    pub max_ms: f64,
117    pub p50_ms: f64,
118    pub p90_ms: f64,
119    pub p99_ms: f64,
120}
121
122impl GapStats {
123    /// 2026-09-26: `stability = (p99 − p50) / p50`: the arrival-gap tail
124    /// spread relative to its median. Dimensionless, so it compares runs
125    /// whose absolute step times differ.
126    ///
127    /// Lower is better: it is a dispersion measure, and 0 means no tail at
128    /// all. `None` unless p50 is positive and finite and p99 is finite.
129    pub fn stability(&self) -> Option<f64> {
130        (self.p50_ms.is_finite() && self.p50_ms > 0.0 && self.p99_ms.is_finite())
131            .then(|| (self.p99_ms - self.p50_ms) / self.p50_ms)
132    }
133
134    /// 2026-09-26: Coefficient of variation, `σ / mean`. Also a dispersion
135    /// measure: lower is better. `None` unless the mean is positive.
136    pub fn cv(&self) -> Option<f64> {
137        (self.mean_ms > 0.0).then(|| self.stddev_ms / self.mean_ms)
138    }
139
140    /// 2026-09-26: The record keys, under `prefix` (e.g. `"c8_"` or `""`).
141    /// The primitives always; the two derived ratios only when defined.
142    pub fn metrics(&self, prefix: &str, m: &mut BTreeMap<String, f64>) {
143        let put = |m: &mut BTreeMap<String, f64>, k: &str, v: f64| {
144            m.insert(format!("{prefix}{k}"), v);
145        };
146        put(m, "arrival_gap_count", self.count as f64);
147        put(m, "arrival_gap_retained", self.retained as f64);
148        put(m, "arrival_gap_mean_ms", self.mean_ms);
149        put(m, "arrival_gap_stddev_ms", self.stddev_ms);
150        put(m, "arrival_gap_max_ms", self.max_ms);
151        put(m, "arrival_gap_p50_ms", self.p50_ms);
152        put(m, "arrival_gap_p90_ms", self.p90_ms);
153        put(m, "arrival_gap_p99_ms", self.p99_ms);
154        if let Some(s) = self.stability() {
155            put(m, "stability", s);
156        }
157        if let Some(cv) = self.cv() {
158            put(m, "arrival_gap_cv", cv);
159        }
160    }
161}
162
163/// 2026-09-26: Inter-token latency: `decode_window_ms / (output_tokens − 1)`.
164/// The one definition both clocks use: the client passes its first-delta →
165/// stream-end window, and `ChatOutcome::server_tpot_ms` passes the
166/// endpoint's `usage.decode_time_ms`.
167///
168/// `None`, never 0, below two output tokens and for a window that is not
169/// positive and finite: a zero would read as an infinitely fast decode.
170pub fn itl_ms(decode_window_ms: f64, output_tokens: usize) -> Option<f64> {
171    (output_tokens >= 2 && decode_window_ms.is_finite() && decode_window_ms > 0.0)
172        .then(|| decode_window_ms / (output_tokens - 1) as f64)
173}
174
175#[cfg(test)]
176#[path = "gaps_tests.rs"]
177mod tests;
metrale_bench/http/gaps.rs

metrale_bench/http/
gaps.rs
