metrale_bench/benchmarks/
stats.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Shared measurement helpers: accept length, prompt synthesis,
4//! percentiles, median and formatting.
5//!
6//! Owner: bench.
7//! Invariants:
8//! - `percentile` uses the nearest-rank rule of `pct` in
9//!   `bench/bench_concurrency.py`, over the finite values only.
10//! - `make_prompt` with an empty tag uses that script's word list and word
11//!   count. Checked 2026-09-26 by running both: the texts are equal up to an
12//!   ISL of 1104; above it the script emits fewer words than it asks for.
13
14use std::fmt::Write as _;
15
16/// 2026-09-26: Emitted tokens per decode step, `completion / (completion -
17/// accepted)`: 1.0 when no draft token was accepted. `decode_floor` and
18/// `concurrency` both read this one rule.
19///
20/// `None` when it cannot be derived: no accept count on the wire, or
21/// `accepted >= completion`. A `None` is not 1.0, which would claim the
22/// server accepted nothing.
23pub fn accept_len(
24    completion_tokens: usize,
25    accepted_prediction_tokens: Option<usize>,
26) -> Option<f64> {
27    let accepted = accepted_prediction_tokens?;
28    (accepted < completion_tokens && completion_tokens > 0)
29        .then(|| completion_tokens as f64 / (completion_tokens - accepted) as f64)
30}
31
32/// 2026-09-26: The filler sentences, the word list of `bench/bench_concurrency.py`'s
33/// `make_prompt`.
34const FILLER: &str = concat!(
35    "The quick brown fox jumped over the lazy dog near a river bank. ",
36    "Mountains rise above the clouds while birds sing their morning songs. ",
37    "Science explores the universe through careful observation and experiment. ",
38    "Ancient civilizations built remarkable structures that still stand today. ",
39    "Music fills the air with rhythm and harmony across every culture. ",
40    "Technology advances rapidly changing how people communicate and work. ",
41    "Forests provide shelter for countless species of plants and animals. ",
42    "Ocean waves crash upon the shore under the light of the moon. ",
43);
44
45/// 2026-09-26: Should the prompt push the model to fill the output budget?
46#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)]
47pub enum PromptMode {
48    /// 2026-09-26: No instruction appended; the model stops when it chooses.
49    Natural,
50    /// 2026-09-26: Append a counting instruction, so replies run toward the
51    /// output budget. The default.
52    #[default]
53    Count,
54}
55
56impl PromptMode {
57    pub fn parse(s: &str) -> Option<Self> {
58        match s.to_ascii_lowercase().as_str() {
59            "natural" | "hello" => Some(PromptMode::Natural),
60            "count" => Some(PromptMode::Count),
61            _ => None,
62        }
63    }
64}
65
66/// 2026-09-26: Build a prompt of roughly `isl_tokens` tokens. A non-empty
67/// `prefix_tag` opens the prompt as `[tag] `: a unique tag gives a prompt no
68/// earlier request shares, and a constant tag gives the same prompt every time.
69pub fn make_prompt(isl_tokens: usize, mode: PromptMode, prefix_tag: &str) -> String {
70    // 2026-09-26: 12 tokens are left for the chat template, as in
71    // `bench/bench_concurrency.py`.
72    let needed = isl_tokens.saturating_sub(12).max(1);
73    let words: Vec<&str> = FILLER.split_whitespace().collect();
74    let mut out = String::with_capacity(needed * 6 + prefix_tag.len() + 80);
75    if !prefix_tag.is_empty() {
76        let _ = write!(out, "[{prefix_tag}] ");
77    }
78    for i in 0..needed {
79        if i > 0 {
80            out.push(' ');
81        }
82        out.push_str(words[i % words.len()]);
83    }
84    if mode == PromptMode::Count {
85        out.push_str(" Count from 1 upward, one number per line, until told to stop.");
86    }
87    out
88}
89
90/// 2026-09-26: `p`-th percentile (0–100) of `values`, nearest rank:
91/// `idx = min(int(n*p/100 + 0.5), n-1)` over the sorted finite values.
92pub fn percentile(values: &[f64], p: u32) -> Option<f64> {
93    if values.is_empty() {
94        return None;
95    }
96    let mut sorted: Vec<f64> = values.iter().copied().filter(|v| v.is_finite()).collect();
97    if sorted.is_empty() {
98        return None;
99    }
100    sorted.sort_by(|a, b| a.partial_cmp(b).expect("filtered to finite"));
101    let n = sorted.len();
102    let idx = ((n as f64 * p as f64 / 100.0) + 0.5) as usize;
103    Some(sorted[idx.min(n - 1)])
104}
105
106/// 2026-09-26: True median of the finite values: the middle element for odd `n`,
107/// the mean of the two middle elements for even `n`. Not `percentile(values,
108/// 50)`, whose index for n=3 is 2, the maximum.
109pub fn median(values: &[f64]) -> Option<f64> {
110    let mut sorted: Vec<f64> = values.iter().copied().filter(|v| v.is_finite()).collect();
111    if sorted.is_empty() {
112        return None;
113    }
114    sorted.sort_by(|a, b| a.partial_cmp(b).expect("filtered to finite"));
115    let n = sorted.len();
116    Some(if n % 2 == 1 {
117        sorted[n / 2]
118    } else {
119        sorted[n / 2 - 1].midpoint(sorted[n / 2])
120    })
121}
122
123/// 2026-09-26: p50 / p90 / p99, each from `percentile`.
124#[derive(Clone, Copy, Debug, Default, PartialEq)]
125pub struct Percentiles {
126    pub p50: Option<f64>,
127    pub p90: Option<f64>,
128    pub p99: Option<f64>,
129}
130
131impl Percentiles {
132    pub fn of(values: &[f64]) -> Self {
133        Self {
134            p50: percentile(values, 50),
135            p90: percentile(values, 90),
136            p99: percentile(values, 99),
137        }
138    }
139}
140
141/// 2026-09-26: Format an optional millisecond value for a table cell: seconds
142/// from 10 s up, and `—` for none, negative or non-finite.
143pub fn fmt_ms(v: Option<f64>) -> String {
144    match v {
145        Some(ms) if !ms.is_finite() || ms < 0.0 => "—".into(),
146        Some(ms) if ms >= 10_000.0 => format!("{:.1}s", ms / 1000.0),
147        Some(ms) => format!("{ms:.1}"),
148        None => "—".into(),
149    }
150}
151
152/// 2026-09-26: Relative change `new` vs `base`, in percent. `None` when either is
153/// non-finite, `base` is not positive, or the result is not finite.
154pub fn pct_delta(new: Option<f64>, base: Option<f64>) -> Option<f64> {
155    match (new, base) {
156        (Some(n), Some(b)) if n.is_finite() && b.is_finite() && b > f64::EPSILON => {
157            let delta = (n - b) / b * 100.0;
158            delta.is_finite().then_some(delta)
159        }
160        _ => None,
161    }
162}
163
164#[cfg(test)]
165mod tests {
166    use super::*;
167    use sha2::{Digest, Sha256};
168
169    fn sha256(text: &str) -> String {
170        metrale_closure::hex_lower(&Sha256::digest(text.as_bytes()))
171    }
172
173    #[test]
174    fn percentile_matches_the_python_nearest_rank_rule() {
175        let v: Vec<f64> = (1..=10).map(|i| i as f64).collect();
176        assert_eq!(percentile(&v, 50), Some(6.0));
177        assert_eq!(percentile(&v, 90), Some(10.0));
178        assert_eq!(percentile(&v, 100), Some(10.0));
179        assert_eq!(percentile(&[], 50), None);
180    }
181
182    #[test]
183    fn percentiles_ignore_non_finite_samples() {
184        let v = vec![1.0, f64::NAN, 3.0, f64::INFINITY];
185        assert_eq!(
186            Percentiles::of(&v),
187            Percentiles {
188                p50: Some(3.0),
189                p90: Some(3.0),
190                p99: Some(3.0),
191            }
192        );
193        assert_eq!(percentile(&[f64::NAN], 50), None);
194    }
195
196    #[test]
197    fn median_handles_empty_odd_even_and_large_finite_samples() {
198        assert_eq!(median(&[]), None);
199        assert_eq!(median(&[f64::NAN, f64::INFINITY]), None);
200        assert_eq!(median(&[3.0, 1.0, 2.0]), Some(2.0));
201        assert_eq!(median(&[4.0, 1.0, 3.0, 2.0]), Some(2.5));
202        assert_eq!(median(&[f64::MAX, f64::MAX]), Some(f64::MAX));
203    }
204
205    #[test]
206    fn prompt_length_tracks_the_request_and_the_tag_changes_the_text() {
207        let a = make_prompt(256, PromptMode::Natural, "");
208        let b = make_prompt(1024, PromptMode::Natural, "");
209        assert!(b.len() > a.len());
210        assert_eq!(a.split_whitespace().count(), 256 - 12);
211        assert!(
212            make_prompt(256, PromptMode::Natural, "s1").starts_with("[s1] The quick brown fox")
213        );
214        assert_eq!(
215            make_prompt(256, PromptMode::Natural, "s1"),
216            make_prompt(256, PromptMode::Natural, "s1")
217        );
218        assert_ne!(
219            make_prompt(256, PromptMode::Natural, "s1"),
220            make_prompt(256, PromptMode::Natural, "s2")
221        );
222    }
223
224    #[test]
225    fn prompt_modes_pin_the_complete_natural_and_forced_bytes() {
226        assert_eq!(
227            sha256(&make_prompt(64, PromptMode::Natural, "fixture")),
228            "1d34b0e6f6c8b13f614be586f7f29ab9fcf918ed11416a3c62702d4a05bf7a54"
229        );
230        assert_eq!(
231            sha256(&make_prompt(64, PromptMode::Count, "fixture")),
232            "619330d2a0c46380c7c3e815226610d5cdea78b760736f496f64a6ec8a6efe96"
233        );
234    }
235
236    #[test]
237    fn millisecond_formatting_rejects_invalid_durations() {
238        assert_eq!(fmt_ms(None), "—");
239        assert_eq!(fmt_ms(Some(12.25)), "12.2");
240        assert_eq!(fmt_ms(Some(10_000.0)), "10.0s");
241        for invalid in [-1.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
242            assert_eq!(fmt_ms(Some(invalid)), "—", "value={invalid:?}");
243        }
244    }
245
246    #[test]
247    fn pct_delta_is_none_when_there_is_no_usable_baseline() {
248        assert_eq!(pct_delta(Some(110.0), Some(100.0)), Some(10.0));
249        assert_eq!(pct_delta(Some(110.0), None), None);
250        assert_eq!(pct_delta(Some(110.0), Some(0.0)), None);
251        assert_eq!(pct_delta(Some(110.0), Some(-100.0)), None);
252        assert_eq!(pct_delta(Some(110.0), Some(f64::INFINITY)), None);
253        assert_eq!(pct_delta(Some(f64::INFINITY), Some(100.0)), None);
254        assert_eq!(pct_delta(Some(f64::NAN), Some(100.0)), None);
255        assert_eq!(pct_delta(Some(f64::MAX), Some(1.0)), None);
256    }
257}
metrale_bench/benchmarks/stats.rs

metrale_bench/benchmarks/
stats.rs
