metrale_bench/benchmarks/
mod.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The benchmark drivers, and the text helpers they share.
4//!
5//! Owner: bench.
6//! Invariants: none beyond the types.
7
8pub mod agentic;
9pub mod baseline;
10pub mod bfcl;
11pub mod concurrency;
12pub mod contamination;
13pub mod decode_floor;
14pub mod default_tier_boot;
15pub mod kat_equality;
16pub mod media_integrity;
17pub mod mlperf_agentic;
18pub mod quick_speed;
19pub mod scheduler_equivalence;
20pub mod serve_matrix;
21pub mod spec_cost;
22pub mod ssm_poison;
23pub mod stats;
24pub mod transcript;
25pub mod ttft;
26pub mod video;
27pub mod vision;
28
29/// 2026-09-26: Collapse a message onto one line (whitespace runs become one
30/// space) and bound it to 300 characters, the last an ellipsis.
31///
32/// Log lines land in a fixed-height pane; a model reply or a traceback pasted
33/// in raw scrolls everything else off the screen.
34pub fn one_line(text: impl AsRef<str>) -> String {
35    const MAX: usize = 300;
36    let mut s: String = text
37        .as_ref()
38        .chars()
39        .map(|c| {
40            if c == '\n' || c == '\r' || c == '\t' {
41                ' '
42            } else {
43                c
44            }
45        })
46        .collect();
47    let squashed = s.split_whitespace().collect::<Vec<_>>().join(" ");
48    s = squashed;
49    if s.chars().count() > MAX {
50        s = s.chars().take(MAX - 1).collect::<String>() + "…";
51    }
52    s
53}
54
55/// 2026-09-26: A prompt prefix tag of the form `<prefix>-<pid>-<run_id>`.
56///
57/// `run_id` comes from the run's [`crate::PluginHandle`], which the executor
58/// numbers per run; with a `prefix` that is unique within the run, the tag
59/// differs from every other run's in the process, and the pid separates
60/// processes. The cold-TTFT measurement (`ttft.rs`) uses it so no two
61/// requests share a cached prefix.
62pub fn unique_prefix_tag(prefix: &str, run_id: u64) -> String {
63    format!("{prefix}-{}-{run_id}", std::process::id())
64}
65
66/// 2026-09-26: Content-derived identity (64-bit FNV-1a over each name and its
67/// bytes) for a named set of committed benchmark assets.
68pub(crate) fn content_stamp<'a>(
69    prefix: &str,
70    assets: impl IntoIterator<Item = (&'a str, &'a [u8])>,
71) -> String {
72    let mut acc: u64 = 1469598103934665603;
73    for (name, bytes) in assets {
74        for byte in name.as_bytes().iter().chain(bytes) {
75            acc ^= *byte as u64;
76            acc = acc.wrapping_mul(1099511628211);
77        }
78    }
79    format!("{prefix}-{acc:016x}")
80}
81
82/// 2026-09-26: Byte offset of the first occurrence of `term` that does not
83/// continue a word on either side: the neighbour is not `_`, and not
84/// alphanumeric (only not numeric, when the term's edge character is
85/// numeric).
86pub(crate) fn first_standalone_term(haystack: &str, term: &str) -> Option<usize> {
87    let first = term.chars().next()?;
88    let last = term.chars().next_back()?;
89    let continues = |edge: char, adjacent: char| {
90        adjacent == '_'
91            || if edge.is_numeric() {
92                adjacent.is_numeric()
93            } else {
94                adjacent.is_alphanumeric()
95            }
96    };
97    haystack.match_indices(term).find_map(|(at, _)| {
98        let before = haystack[..at].chars().next_back();
99        let after = haystack[at + term.len()..].chars().next();
100        (!before.is_some_and(|adjacent| continues(first, adjacent))
101            && !after.is_some_and(|adjacent| continues(last, adjacent)))
102        .then_some(at)
103    })
104}
105
106#[cfg(test)]
107mod tests {
108    use super::*;
109
110    #[test]
111    fn one_line_squashes_and_truncates() {
112        assert_eq!(one_line(" a\n b\r\t\tc \u{a0} d "), "a b c d");
113        assert_eq!(one_line("é".repeat(299)), "é".repeat(299));
114        assert_eq!(one_line("é".repeat(300)), "é".repeat(300));
115        assert_eq!(
116            one_line("é".repeat(301)),
117            "é".repeat(299) + "…",
118            "the ellipsis is the 300th character"
119        );
120    }
121
122    #[test]
123    fn prefix_tags_never_repeat() {
124        let a = unique_prefix_tag("cold", 1);
125        let b = unique_prefix_tag("cold", 2);
126        let c = unique_prefix_tag("warm", 1);
127        assert_eq!(a, format!("cold-{}-1", std::process::id()));
128        assert_ne!(a, b);
129        assert_ne!(a, c);
130    }
131}
metrale_bench/benchmarks/mod.rs

metrale_bench/benchmarks/
mod.rs
