metrale_bench/benchmarks/vision/
request.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Building an image chat request, and reading the vision-token
4//! count back out.
5//!
6//! Owner: bench, vision.
7//! Invariants: none beyond the types.
8
9use anyhow::{Context, Result};
10use base64::Engine;
11use serde_json::{Value, json};
12
13/// 2026-09-26: A fixture as a base64 `data:` URI, labelled `image/png`
14/// whatever its bytes; the server picks the decoder from the bytes. The server
15/// fetches `http(s)` image URLs only with `--vision-allow-remote-images`, so a
16/// data URI works on every serve and keeps the network out of the run.
17pub fn data_uri(png: &[u8]) -> String {
18    let mut s = String::from("data:image/png;base64,");
19    base64::engine::general_purpose::STANDARD.encode_string(png, &mut s);
20    s
21}
22
23/// 2026-09-26: One streamed chat request carrying `images`, in order, then
24/// `prompt`.
25///
26/// Temperature 0, since the assertions are about what the model saw. Thinking
27/// is off: a reasoning block can use the whole of `max_tokens` and leave empty
28/// content.
29pub fn body(model: &str, images: &[&[u8]], prompt: &str, max_tokens: usize) -> Value {
30    let mut content: Vec<Value> = images
31        .iter()
32        .map(|png| json!({"type": "image_url", "image_url": {"url": data_uri(png)}}))
33        .collect();
34    content.push(json!({"type": "text", "text": prompt}));
35    json!({
36        "model": model,
37        "stream": true,
38        "temperature": 0.0,
39        "max_tokens": max_tokens,
40        "chat_template_kwargs": {"enable_thinking": false},
41        "messages": [{"role": "user", "content": content}],
42    })
43}
44
45/// 2026-09-26: Vision tokens in a request: `prompt_tokens` minus the template
46/// `overhead`, or an error when `overhead` is larger.
47///
48/// The driver measures `overhead` once per run, from a calibration request
49/// whose vision-token count is known, because it depends on the checkpoint's
50/// chat template.
51pub fn vision_tokens(prompt_tokens: usize, overhead: usize) -> Result<usize> {
52    prompt_tokens.checked_sub(overhead).with_context(|| {
53        format!(
54            "prompt_tokens {prompt_tokens} is below the measured template overhead \
55             {overhead} — the calibration request and this one did not render the same \
56             template, so the subtraction is meaningless"
57        )
58    })
59}
60
61#[cfg(test)]
62#[path = "request_tests.rs"]
63mod request_tests;
metrale_bench/benchmarks/vision/request.rs

metrale_bench/benchmarks/vision/
request.rs
