metrale_bench/benchmarks/video/
concurrency.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The concurrency leg shared by the video and vision benchmarks:
4//! for each of `LEVELS`, that many copies of one request are sent at once and
5//! scored for how many returned, how many were correct, and whether they
6//! agree on one prompt-token count.
7//!
8//! It checks correctness, not throughput: `wall_ms` is recorded and never
9//! read by `ok`, `ok_against` or `sweep_ok`. The copies are identical, so this
10//! leg cannot see one request answered with another's input;
11//! `media_integrity::heterogeneous_concurrency` covers that.
12//!
13//! Owner: bench, video and vision.
14//! Invariants: none beyond the types.
15
16use std::time::{Duration, Instant};
17
18use crate::http::{self, ChatOutcome};
19use crate::plugin::PluginHandle;
20
21/// 2026-09-26: The levels swept, in order. `sweep_ok` needs one result per
22/// level, and the first (C=1) sets the prompt-token baseline.
23pub const LEVELS: &[usize] = &[1, 2, 4];
24
25/// 2026-09-26: One level's outcome.
26#[derive(Debug, Clone, PartialEq, Eq)]
27pub struct LevelResult {
28    pub conc: usize,
29    /// 2026-09-26: Replies that came back without an error.
30    pub returned: usize,
31    /// 2026-09-26: Returned replies for which `is_correct` held.
32    pub correct: usize,
33    /// 2026-09-26: Distinct prompt-token counts among the returned replies;
34    /// `ok` requires exactly one.
35    pub distinct_token_counts: usize,
36    /// 2026-09-26: The agreed prompt-token count when there was exactly one,
37    /// kept so later levels can be compared with the C=1 level.
38    pub prompt_tokens: Option<usize>,
39    pub wall_ms: u128,
40    pub errors: Vec<String>,
41}
42
43impl LevelResult {
44    pub fn ok(&self) -> bool {
45        self.correct == self.conc && self.distinct_token_counts == 1 && self.prompt_tokens.is_some()
46    }
47
48    /// 2026-09-26: `ok`, and the agreed prompt-token count equals the C=1
49    /// level's.
50    pub fn ok_against(&self, baseline_prompt_tokens: usize) -> bool {
51        self.ok() && self.prompt_tokens == Some(baseline_prompt_tokens)
52    }
53
54    pub fn geometry_detail(&self, baseline_prompt_tokens: Option<usize>) -> String {
55        match (self.prompt_tokens, baseline_prompt_tokens) {
56            (Some(got), Some(want)) if got != want => {
57                format!("{got} prompt tokens, C=1 baseline {want}")
58            }
59            (Some(got), _) => format!("one geometry ({got} prompt tokens)"),
60            (None, _) => format!("{} distinct token counts", self.distinct_token_counts),
61        }
62    }
63}
64
65/// 2026-09-26: True when there is one result per `LEVELS` entry, in order,
66/// and each is `ok_against` the first result's prompt-token count.
67pub fn sweep_ok(results: &[LevelResult]) -> bool {
68    let Some(baseline_prompt_tokens) = results.first().and_then(|r| r.prompt_tokens) else {
69        return false;
70    };
71    results.len() == LEVELS.len()
72        && results
73            .iter()
74            .zip(LEVELS)
75            .all(|(r, &level)| r.conc == level && r.ok_against(baseline_prompt_tokens))
76}
77
78/// 2026-09-26: Send `conc` copies of `body` at once and score them with
79/// `is_correct`.
80pub async fn run_level(
81    handle: &PluginHandle,
82    body: &serde_json::Value,
83    conc: usize,
84    timeout: Duration,
85    // 2026-09-26: `Sync` as well as `Fn`: the reference is held across an
86    // await inside a `Send` benchmark future, and `&dyn Fn` is `Send` only
87    // when the closure type is `Sync`.
88    is_correct: &(dyn Fn(&str) -> bool + Sync),
89) -> LevelResult {
90    let start = Instant::now();
91    let futures: Vec<_> = (0..conc)
92        .map(|_| http::chat_stream(handle.target(), body, timeout))
93        .collect();
94    let outcomes: Vec<anyhow::Result<ChatOutcome>> = futures::future::join_all(futures).await;
95    let wall_ms = start.elapsed().as_millis();
96
97    let mut returned = 0usize;
98    let mut correct = 0usize;
99    let mut counts: Vec<usize> = Vec::new();
100    let mut errors = Vec::new();
101    for o in outcomes {
102        match o {
103            Ok(out) => {
104                returned += 1;
105                counts.push(out.prompt_tokens);
106                if is_correct(out.text.trim()) {
107                    correct += 1;
108                }
109            }
110            Err(e) => errors.push(crate::benchmarks::one_line(format!("{e:#}"))),
111        }
112    }
113    counts.sort_unstable();
114    counts.dedup();
115    let prompt_tokens = (counts.len() == 1).then(|| counts[0]);
116    LevelResult {
117        conc,
118        returned,
119        correct,
120        distinct_token_counts: counts.len(),
121        prompt_tokens,
122        wall_ms,
123        errors,
124    }
125}
126
127#[cfg(test)]
128#[path = "concurrency_tests.rs"]
129mod concurrency_tests;
metrale_bench/benchmarks/video/concurrency.rs

metrale_bench/benchmarks/video/
concurrency.rs
