metrale_bench/benchmarks/vision/
probes.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Capability probes: did the model see the picture? Each probe
4//! attaches fixtures, asks a question, and is scored by
5//! `score::reply_matches` against terms the reply must and must not contain.
6//! The fixtures they use come from `scripts/gen_test_images.py`, which draws a
7//! size label on each.
8//!
9//! This leg shows that vision ran; `geometry.rs` checks the token count.
10//! Geometry alone would pass an encoder that produced the right count from
11//! wrong embeddings.
12//!
13//! Owner: bench, vision.
14//! Invariants: none beyond the types.
15
16/// 2026-09-26: A probe: which fixtures to attach, what to ask, what a correct
17/// answer must and must not contain.
18pub struct Probe {
19    pub id: &'static str,
20    /// 2026-09-26: Fixture names, looked up in `provision::FIXTURES`. Empty
21    /// sends no image; see [`CONTROL`].
22    pub images: &'static [&'static str],
23    pub prompt: &'static str,
24    /// 2026-09-26: Lowercase terms that must all appear.
25    pub want_all: &'static [&'static str],
26    /// 2026-09-26: Lowercase terms none of which may appear, so a reply that
27    /// names every option does not pass.
28    pub want_none: &'static [&'static str],
29}
30
31pub const PROBES: &[Probe] = &[
32    Probe {
33        id: "sees-an-image",
34        images: &["01_square_224.png"],
35        prompt: "Describe what you see in this image in one short sentence.",
36        want_all: &[],
37        // 2026-09-26: Nothing positive is asserted; the reply must not say it
38        // sees no image.
39        want_none: &["cannot see", "no image", "unable to see", "don't see"],
40    },
41    Probe {
42        id: "reads-the-size-label",
43        images: &["07_hd_1280x720.png"],
44        prompt: "This image has a size label drawn on it. Read the label exactly.",
45        want_all: &["1280"],
46        want_none: &["cannot see", "no image"],
47    },
48    Probe {
49        id: "multi-image-order",
50        images: &["01_square_224.png", "08_portrait_480x854.png"],
51        // 2026-09-26: The question is about the first image, so the answer
52        // depends on the images arriving in order.
53        prompt: "You are shown two images. Is the FIRST one square or portrait? \
54                 Answer with one word.",
55        want_all: &["square"],
56        want_none: &["portrait"],
57    },
58];
59
60/// 2026-09-26: The no-image control: the size-label question with nothing
61/// attached. It passes when the reply does not contain `1280`, the term
62/// `reads-the-size-label` requires; otherwise the capability leg is not
63/// evidence and the run is VACUOUS.
64pub const CONTROL: Probe = Probe {
65    id: "control-no-image",
66    images: &[],
67    prompt: "This image has a size label drawn on it. Read the label exactly.",
68    want_all: &[],
69    want_none: &["1280"],
70};
71
72/// 2026-09-26: The probe the concurrency leg sends, `reads-the-size-label`, so
73/// a reply passes only by reading the label from the image.
74pub fn concurrency_probe() -> &'static Probe {
75    PROBES
76        .iter()
77        .find(|probe| probe.id == "reads-the-size-label")
78        .expect("reads-the-size-label probe is part of the fixed probe set")
79}
80
81#[cfg(test)]
82#[path = "probes_tests.rs"]
83mod probes_tests;
metrale_bench/benchmarks/vision/probes.rs

metrale_bench/benchmarks/vision/
probes.rs
