metrale_bench/benchmarks/
media_integrity.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Cross-request and cross-path integrity checks, used by the
4//! vision and video benchmarks. They target machinery a single well-formed
5//! request cannot reach, whose failure is a fluent answer to the wrong input
6//! rather than an error:
7//!
8//! * [`heterogeneous_concurrency`]: several different requests in flight at
9//!   once, each required to get its own answer.
10//! * [`cache_leak`]: the same prompt text with a different image, back to
11//!   back.
12//! * [`long_prompt_path`]: the same image and question with filler prepended.
13//! * [`media_in_history`]: media in an earlier turn or a tool result.
14//! * [`stream_parity`]: the same request streamed and blocking.
15//! * [`multi_choice`]: `n > 1` with an image.
16//! * `responses_parity` and `thinking_parity` (the `responses` submodule).
17//!
18//! Owner: bench, media integrity.
19//! Invariants: a check whose request failed never returns `Cell::Pass`.
20
21use std::time::Duration;
22
23use serde_json::{Value, json};
24
25use crate::http;
26use crate::plugin::PluginHandle;
27
28/// 2026-09-26: One concurrent subject: the request, a predicate scoring its
29/// own reply, and a label for the failure message.
30pub type Subject = (Value, Box<dyn Fn(&str) -> bool + Send + Sync>, String);
31
32/// 2026-09-26: Outcome of one integrity check.
33#[derive(Debug, Clone, PartialEq, Eq)]
34pub enum Cell {
35    Pass { id: &'static str, detail: String },
36    Fail { id: &'static str, detail: String },
37    Skipped { id: &'static str, why: String },
38    Error { id: &'static str, msg: String },
39}
40
41impl Cell {
42    pub fn id(&self) -> &'static str {
43        match self {
44            Cell::Pass { id, .. }
45            | Cell::Fail { id, .. }
46            | Cell::Skipped { id, .. }
47            | Cell::Error { id, .. } => id,
48        }
49    }
50    pub fn passed(&self) -> bool {
51        matches!(self, Cell::Pass { .. })
52    }
53    pub fn measured(&self) -> bool {
54        matches!(self, Cell::Pass { .. } | Cell::Fail { .. })
55    }
56    pub fn line(&self) -> String {
57        match self {
58            Cell::Pass { id, detail } => format!("{id}: {detail}"),
59            Cell::Fail { id, detail } => format!("{id}: FAILED — {detail}"),
60            Cell::Skipped { id, why } => format!("{id}: skipped — {why}"),
61            Cell::Error { id, msg } => format!("{id}: {msg}"),
62        }
63    }
64}
65
66/// 2026-09-26: One image part plus a prompt, as a streaming chat body with
67/// thinking off.
68pub fn image_request(
69    model: &str,
70    mime: &str,
71    bytes: &[u8],
72    prompt: &str,
73    max_tokens: usize,
74) -> Value {
75    use base64::Engine;
76    let mut uri = format!("data:{mime};base64,");
77    base64::engine::general_purpose::STANDARD.encode_string(bytes, &mut uri);
78    json!({
79        "model": model,
80        "stream": true,
81        "temperature": 0.0,
82        "max_tokens": max_tokens,
83        "chat_template_kwargs": {"enable_thinking": false},
84        "messages": [{"role": "user", "content": [
85            {"type": "image_url", "image_url": {"url": uri}},
86            {"type": "text", "text": prompt},
87        ]}],
88    })
89}
90
91/// 2026-09-26: Several different requests at once, each scored against its own
92/// expected answer.
93///
94/// With `METRALE_VISION_CODISPATCH` on, image requests admitted in one tick
95/// share one encoder output and each reads its slice through per-request base
96/// offsets (`ModelVision::set_vision_slice_base`). When every request is
97/// identical, every offset is interchangeable and an off-by-one is invisible;
98/// with different content, a mis-sliced offset hands request A the answer to
99/// request B. Fewer than two subjects is a skip.
100pub async fn heterogeneous_concurrency(
101    handle: &PluginHandle,
102    subjects: Vec<Subject>,
103    timeout: Duration,
104) -> Cell {
105    const ID: &str = "heterogeneous-concurrency";
106    if subjects.len() < 2 {
107        return Cell::Skipped {
108            id: ID,
109            why: "fewer than two subjects".to_string(),
110        };
111    }
112    let n = subjects.len();
113    let futures: Vec<_> = subjects
114        .iter()
115        .map(|(body, _, _)| http::chat_stream(handle.target(), body, timeout))
116        .collect();
117    let outs = futures::future::join_all(futures).await;
118
119    let mut wrong = Vec::new();
120    let mut errors = Vec::new();
121    for (out, (_, want, label)) in outs.into_iter().zip(subjects.iter()) {
122        match out {
123            Ok(o) => {
124                if !want(o.text.trim()) {
125                    wrong.push(format!(
126                        "{label} got \"{}\"",
127                        crate::benchmarks::one_line(o.text.chars().take(60).collect::<String>())
128                    ));
129                }
130            }
131            Err(e) => errors.push(format!(
132                "{label}: {}",
133                crate::benchmarks::one_line(format!("{e:#}"))
134            )),
135        }
136    }
137    if !errors.is_empty() {
138        return Cell::Error {
139            id: ID,
140            msg: errors.join("; "),
141        };
142    }
143    if wrong.is_empty() {
144        Cell::Pass {
145            id: ID,
146            detail: format!("{n} different requests in flight, each got its own answer"),
147        }
148    } else {
149        Cell::Fail {
150            id: ID,
151            detail: format!(
152                "{}/{n} replies did not match their own input — {}",
153                wrong.len(),
154                wrong.join("; ")
155            ),
156        }
157    }
158}
159
160/// 2026-09-26: The same prompt text with a different image, back to back.
161///
162/// Vision prompts must not be served from the prefix cache. The model-engine
163/// guard is `tokens_have_vision_pad`, which must recognise both the image and
164/// the video pad; if it missed one, the second request would match the
165/// first's cached prefix and answer about the previous image. `first_marker`
166/// tells that case apart from a merely wrong reply.
167pub async fn cache_leak(
168    handle: &PluginHandle,
169    first: Value,
170    second: Value,
171    second_want: &(dyn Fn(&str) -> bool + Sync),
172    first_marker: &(dyn Fn(&str) -> bool + Sync),
173    timeout: Duration,
174) -> Cell {
175    const ID: &str = "prefix-cache-isolation";
176    let a = match http::chat_stream(handle.target(), &first, timeout).await {
177        Ok(o) => o,
178        Err(e) => {
179            return Cell::Error {
180                id: ID,
181                msg: crate::benchmarks::one_line(format!("{e:#}")),
182            };
183        }
184    };
185    let b = match http::chat_stream(handle.target(), &second, timeout).await {
186        Ok(o) => o,
187        Err(e) => {
188            return Cell::Error {
189                id: ID,
190                msg: crate::benchmarks::one_line(format!("{e:#}")),
191            };
192        }
193    };
194    let b_text = b.text.trim();
195    if second_want(b_text) {
196        return Cell::Pass {
197            id: ID,
198            detail: format!(
199                "identical prompt, different media: second reply is its own ({} then {} prompt \
200                 tokens)",
201                a.prompt_tokens, b.prompt_tokens
202            ),
203        };
204    }
205    // 2026-09-26: Distinguish "answered the first image", a cache hit, from
206    // merely wrong.
207    let detail = if first_marker(b_text) {
208        format!(
209            "the second request was answered with the FIRST image's content — the prefix cache \
210             served a vision prompt. Reply: \"{}\"",
211            crate::benchmarks::one_line(b_text.chars().take(80).collect::<String>())
212        )
213    } else {
214        format!(
215            "the second reply matched neither expectation: \"{}\"",
216            crate::benchmarks::one_line(b_text.chars().take(80).collect::<String>())
217        )
218    };
219    Cell::Fail { id: ID, detail }
220}
221
222/// 2026-09-26: The same image and question, short and with a large block of
223/// filler text prepended; both replies must satisfy `want`.
224///
225/// With `METRALE_VISION_CODISPATCH` on, `phase_start_prefills` admits an image
226/// request to the batched encode only when its prompt fits
227/// `max_prefill_tokens`; a longer prompt encodes its own images
228/// (`prepare_vision_embed`), so the two prompts take different paths.
229pub async fn long_prompt_path(
230    handle: &PluginHandle,
231    short: Value,
232    long: Value,
233    want: &(dyn Fn(&str) -> bool + Sync),
234    timeout: Duration,
235) -> Cell {
236    const ID: &str = "long-prompt-splice";
237    let s = match http::chat_stream(handle.target(), &short, timeout).await {
238        Ok(o) => o,
239        Err(e) => {
240            return Cell::Error {
241                id: ID,
242                msg: crate::benchmarks::one_line(format!("{e:#}")),
243            };
244        }
245    };
246    let l = match http::chat_stream(handle.target(), &long, timeout).await {
247        Ok(o) => o,
248        Err(e) => {
249            return Cell::Error {
250                id: ID,
251                msg: crate::benchmarks::one_line(format!("{e:#}")),
252            };
253        }
254    };
255    let short_ok = want(s.text.trim());
256    let long_ok = want(l.text.trim());
257    if short_ok && long_ok {
258        Cell::Pass {
259            id: ID,
260            detail: format!(
261                "same answer on both paths ({} vs {} prompt tokens)",
262                s.prompt_tokens, l.prompt_tokens
263            ),
264        }
265    } else {
266        Cell::Fail {
267            id: ID,
268            detail: format!(
269                "short path {}, long path {} ({} vs {} prompt tokens) — the two splices disagree",
270                if short_ok { "ok" } else { "WRONG" },
271                if long_ok { "ok" } else { "WRONG" },
272                s.prompt_tokens,
273                l.prompt_tokens
274            ),
275        }
276    }
277}
278
279/// 2026-09-26: Media that is not in the message asking the question: in an
280/// earlier turn or in a tool result. The server collects media from every
281/// message, tool results included (`api/chat/msg_entry.rs`,
282/// `collect_message_media`); a regression that scanned only the final message
283/// would pass the legs that put the image and the question in one turn.
284pub async fn media_in_history(
285    handle: &PluginHandle,
286    body: Value,
287    want: &(dyn Fn(&str) -> bool + Sync),
288    id: &'static str,
289    timeout: Duration,
290) -> Cell {
291    match http::chat_stream(handle.target(), &body, timeout).await {
292        Ok(o) => {
293            let text = o.text.trim();
294            if want(text) {
295                Cell::Pass {
296                    id,
297                    detail: format!("answered from history ({} prompt tokens)", o.prompt_tokens),
298                }
299            } else {
300                Cell::Fail {
301                    id,
302                    detail: format!(
303                        "did not answer about the earlier media: \"{}\"",
304                        crate::benchmarks::one_line(text.chars().take(80).collect::<String>())
305                    ),
306                }
307            }
308        }
309        Err(e) => Cell::Error {
310            id,
311            msg: crate::benchmarks::one_line(format!("{e:#}")),
312        },
313    }
314}
315
316/// 2026-09-26: The same request streamed and not streamed.
317///
318/// The blocking path builds its response separately (`api/chat_blocking.rs`),
319/// so a server can stream correctly and assemble a blocking reply wrongly.
320/// Both must answer correctly and agree on `prompt_tokens`: the request is
321/// the same apart from `stream`, so a difference there means the two paths
322/// built different prompts from it.
323pub async fn stream_parity(
324    handle: &PluginHandle,
325    mut body: Value,
326    want: &(dyn Fn(&str) -> bool + Sync),
327    timeout: Duration,
328) -> Cell {
329    const ID: &str = "stream-blocking-parity";
330    body["stream"] = Value::Bool(true);
331    let streamed = match http::chat_stream(handle.target(), &body, timeout).await {
332        Ok(o) => o,
333        Err(e) => {
334            return Cell::Error {
335                id: ID,
336                msg: crate::benchmarks::one_line(format!("streaming: {e:#}")),
337            };
338        }
339    };
340    body["stream"] = Value::Bool(false);
341    let blocking = match http::chat_blocking(handle.target(), &body, timeout).await {
342        Ok(o) => o,
343        Err(e) => {
344            return Cell::Error {
345                id: ID,
346                msg: crate::benchmarks::one_line(format!("blocking: {e:#}")),
347            };
348        }
349    };
350    let b_text = blocking.choices.first().cloned().unwrap_or_default();
351    let s_ok = want(streamed.text.trim());
352    let b_ok = want(b_text.trim());
353    if !s_ok || !b_ok {
354        return Cell::Fail {
355            id: ID,
356            detail: format!(
357                "streaming {}, blocking {} — the two response paths disagree about the same \
358                 image",
359                if s_ok { "ok" } else { "WRONG" },
360                if b_ok { "ok" } else { "WRONG" }
361            ),
362        };
363    }
364    if streamed.prompt_tokens != blocking.prompt_tokens {
365        return Cell::Fail {
366            id: ID,
367            detail: format!(
368                "both answered correctly but built DIFFERENT prompts: {} tokens streaming vs {} \
369                 blocking, from a byte-identical request",
370                streamed.prompt_tokens, blocking.prompt_tokens
371            ),
372        };
373    }
374    Cell::Pass {
375        id: ID,
376        detail: format!(
377            "both paths correct and agree on {} prompt tokens",
378            streamed.prompt_tokens
379        ),
380    }
381}
382
383/// 2026-09-26: `n > 1` with an image, on the blocking path.
384///
385/// `api/chat_blocking.rs` gives the image pixels to choice 0 and an empty
386/// vector to every later choice. If a later choice loses the image rather than
387/// sharing choice 0's encode, it answers about nothing. Every choice must be
388/// present and answer about the picture.
389pub async fn multi_choice(
390    handle: &PluginHandle,
391    mut body: Value,
392    n: usize,
393    want: &(dyn Fn(&str) -> bool + Sync),
394    timeout: Duration,
395) -> Cell {
396    const ID: &str = "multi-choice-image";
397    body["stream"] = Value::Bool(false);
398    body["n"] = Value::from(n);
399    match http::chat_blocking(handle.target(), &body, timeout).await {
400        Ok(o) => {
401            if o.choices.len() != n {
402                return Cell::Fail {
403                    id: ID,
404                    detail: format!("asked for n={n}, got {} choices", o.choices.len()),
405                };
406            }
407            let bad: Vec<usize> = o
408                .choices
409                .iter()
410                .enumerate()
411                .filter(|(_, c)| !want(c.trim()))
412                .map(|(i, _)| i)
413                .collect();
414            if bad.is_empty() {
415                Cell::Pass {
416                    id: ID,
417                    detail: format!("all {n} choices answered about the image"),
418                }
419            } else {
420                Cell::Fail {
421                    id: ID,
422                    detail: format!(
423                        "choice(s) {bad:?} did not answer about the image — the later choices \
424                         are not seeing it"
425                    ),
426                }
427            }
428        }
429        // 2026-09-26: `n > 1` may not be supported; an error containing "400"
430        // or "not supported" is a skip rather than a failure.
431        Err(e) => {
432            let msg = crate::benchmarks::one_line(format!("{e:#}"));
433            let unsupported = msg.contains("400") || msg.to_lowercase().contains("not supported");
434            if unsupported {
435                Cell::Skipped { id: ID, why: msg }
436            } else {
437                Cell::Error { id: ID, msg }
438            }
439        }
440    }
441}
442
443#[path = "media_integrity_responses.rs"]
444mod responses;
445pub use responses::{
446    responses_image_request, responses_parity, thinking_image_request, thinking_parity,
447};
448
449#[cfg(test)]
450#[path = "media_integrity_tests.rs"]
451mod media_integrity_tests;
metrale_bench/benchmarks/media_integrity.rs

metrale_bench/benchmarks/
media_integrity.rs
