metrale_bench/benchmarks/video/
request.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Request bodies for the video legs, and the decoder-unavailable
4//! predicate.
5//!
6//! Owner: bench, video.
7//! Invariants: none beyond the types.
8
9use base64::Engine;
10use serde_json::{Value, json};
11
12/// 2026-09-26: The question the ordered-color legs ask. It asks only for the
13/// color names in order, so `score::order_matches` can score the reply.
14pub const ORDER_PROMPT: &str = "This video is a sequence of solid background colors. List the colors in the order they \
15     appear, separated by commas. Answer with only the color names.";
16
17/// 2026-09-26: `bytes` as a base64 `data:` URI with type `mime`.
18pub fn data_uri(mime: &str, bytes: &[u8]) -> String {
19    let mut s = format!("data:{mime};base64,");
20    base64::engine::general_purpose::STANDARD.encode_string(bytes, &mut s);
21    s
22}
23
24/// 2026-09-26: One streamed request carrying a single video, then `prompt`.
25///
26/// Temperature 0 and thinking off, as in the vision benchmark's
27/// `request::body`: the assertions are about what the model saw, and a
28/// reasoning block can use the whole token budget and leave empty content.
29pub fn video_body(model: &str, mime: &str, bytes: &[u8], prompt: &str, max_tokens: usize) -> Value {
30    json!({
31        "model": model,
32        "stream": true,
33        "temperature": 0.0,
34        "max_tokens": max_tokens,
35        "chat_template_kwargs": {"enable_thinking": false},
36        "messages": [{"role": "user", "content": [
37            {"type": "video_url", "video_url": {"url": data_uri(mime, bytes)}},
38            {"type": "text", "text": prompt},
39        ]}],
40    })
41}
42
43/// 2026-09-26: One request carrying an image, then a video, then `prompt`. The
44/// image part is labelled `image/png` whatever its bytes; the server picks the
45/// image decoder from the bytes (`model-layers/src/vision_preprocess.rs`).
46pub fn mixed_body(
47    model: &str,
48    png: &[u8],
49    mime: &str,
50    video: &[u8],
51    prompt: &str,
52    max_tokens: usize,
53) -> Value {
54    json!({
55        "model": model,
56        "stream": true,
57        "temperature": 0.0,
58        "max_tokens": max_tokens,
59        "chat_template_kwargs": {"enable_thinking": false},
60        "messages": [{"role": "user", "content": [
61            {"type": "image_url", "image_url": {"url": data_uri("image/png", png)}},
62            {"type": "video_url", "video_url": {"url": data_uri(mime, video)}},
63            {"type": "text", "text": prompt},
64        ]}],
65    })
66}
67
68/// 2026-09-26: The control: `prompt` alone, as a plain string `content`.
69pub fn text_only_body(model: &str, prompt: &str, max_tokens: usize) -> Value {
70    json!({
71        "model": model,
72        "stream": true,
73        "temperature": 0.0,
74        "max_tokens": max_tokens,
75        "chat_template_kwargs": {"enable_thinking": false},
76        "messages": [{"role": "user", "content": prompt}],
77    })
78}
79
80/// 2026-09-26: One request carrying a single image: with [`text_array_body`],
81/// the two baselines the pad-arithmetic cell subtracts.
82///
83/// Both send `content` as an array, like [`video_body`] and [`mixed_body`], so
84/// every term of the arithmetic goes through the same content shape.
85/// [`text_only_body`] sends a plain string and is not used as a baseline.
86pub fn image_body(model: &str, mime: &str, png: &[u8], prompt: &str, max_tokens: usize) -> Value {
87    json!({
88        "model": model,
89        "stream": true,
90        "temperature": 0.0,
91        "max_tokens": max_tokens,
92        "chat_template_kwargs": {"enable_thinking": false},
93        "messages": [{"role": "user", "content": [
94            {"type": "image_url", "image_url": {"url": data_uri(mime, png)}},
95            {"type": "text", "text": prompt},
96        ]}],
97    })
98}
99
100/// 2026-09-26: Text only, as a content array. See [`image_body`].
101pub fn text_array_body(model: &str, prompt: &str, max_tokens: usize) -> Value {
102    json!({
103        "model": model,
104        "stream": true,
105        "temperature": 0.0,
106        "max_tokens": max_tokens,
107        "chat_template_kwargs": {"enable_thinking": false},
108        "messages": [{"role": "user", "content": [
109            {"type": "text", "text": prompt},
110        ]}],
111    })
112}
113
114/// 2026-09-26: Does this error read as "the server cannot decode that
115/// container"? Such a leg is skipped, not failed: ffmpeg decoding is a
116/// deployment choice (`--video-allow-ffmpeg`).
117///
118/// The strings match the server's messages in
119/// `model-layers/src/video_decode_ffmpeg.rs` (`decode_frames`,
120/// `spawn_failure`, the availability probe). That file's tests pin only the
121/// "is ffmpeg installed" hint, so rewording the others there would turn these
122/// skips into failures without failing a test.
123pub fn is_decoder_unavailable(err: &str) -> bool {
124    let e = err.to_lowercase();
125    let quoted_binary_unavailable = e
126        .split_once(" could not be run:")
127        .is_some_and(|(binary, _)| binary.starts_with('"') && binary.ends_with('"'));
128    e.contains("subprocess decoding is disabled")
129        || (e.contains("could not run ") && e.contains("is ffmpeg installed"))
130        || quoted_binary_unavailable
131}
132
133#[cfg(test)]
134#[path = "request_tests.rs"]
135mod request_tests;
metrale_bench/benchmarks/video/request.rs

metrale_bench/benchmarks/video/
request.rs
