metrale_bench/benchmarks/vision/
score.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The vision benchmark's cells and verdict. A run whose no-image
4//! control did not hold is VACUOUS, whatever the other cells say.
5//!
6//! Owner: bench, vision.
7//! Invariants: none beyond the types.
8
9use std::fmt;
10
11/// 2026-09-26: How one geometry cell came out.
12#[derive(Debug, Clone, PartialEq, Eq)]
13pub enum GeomCell {
14    /// 2026-09-26: The measured vision tokens equal the prediction.
15    Match {
16        fixture: &'static str,
17        tokens: usize,
18    },
19    /// 2026-09-26: The measured vision tokens differ from the prediction.
20    Mismatch {
21        fixture: &'static str,
22        want: usize,
23        got: usize,
24    },
25    /// 2026-09-26: Not asserted: the server refused the image as over its
26    /// encoder's capacity.
27    Unmeasured { fixture: &'static str, why: String },
28    /// 2026-09-26: The request, or the vision-token subtraction, failed.
29    Error { fixture: &'static str, msg: String },
30}
31
32/// 2026-09-26: How one capability probe came out.
33#[derive(Debug, Clone, PartialEq, Eq)]
34pub enum ProbeCell {
35    Pass { id: &'static str },
36    Fail { id: &'static str, reply: String },
37    Error { id: &'static str, msg: String },
38}
39
40#[derive(Debug, Clone, Copy, PartialEq, Eq)]
41pub enum Verdict {
42    Pass,
43    Fail,
44    /// 2026-09-26: The no-image control answered as though it saw an image, so
45    /// the capability probes are not evidence.
46    Vacuous,
47}
48
49impl fmt::Display for Verdict {
50    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
51        f.write_str(match self {
52            Verdict::Pass => "PASS",
53            Verdict::Fail => "FAIL",
54            Verdict::Vacuous => "VACUOUS",
55        })
56    }
57}
58
59/// 2026-09-26: Does a reply satisfy a probe's expectations?
60///
61/// Case-insensitive term matching. A numeric term matches where no digit
62/// touches it; any other term must stand alone as a word
63/// (`first_standalone_term`). Every `want_all` term must match and no
64/// `want_none` term may.
65pub fn reply_matches(reply: &str, want_all: &[&str], want_none: &[&str]) -> bool {
66    let hay = reply.to_lowercase();
67    let contains_term = |term: &str| {
68        if !term.is_empty() && term.chars().all(|ch| ch.is_ascii_digit()) {
69            // 2026-09-26: The HD fixture's label is `07_hd_1280x720  1280x720`
70            // (`scripts/gen_test_images.py`), where letters and `_` touch 1280.
71            hay.match_indices(term).any(|(at, _)| {
72                !hay[..at]
73                    .chars()
74                    .next_back()
75                    .is_some_and(|ch| ch.is_ascii_digit())
76                    && !hay[at + term.len()..]
77                        .chars()
78                        .next()
79                        .is_some_and(|ch| ch.is_ascii_digit())
80            })
81        } else {
82            crate::benchmarks::first_standalone_term(&hay, term).is_some()
83        }
84    };
85    want_all.iter().all(|term| contains_term(term))
86        && !want_none.iter().any(|term| contains_term(term))
87}
88
89/// 2026-09-26: Fold the geometry and probe cells into one verdict: Vacuous
90/// when the control did not hold, whatever else happened; otherwise Fail on
91/// any geometry `Mismatch` or `Error` or any probe that did not pass; otherwise
92/// Pass. `Unmeasured` cells do not fail the run.
93pub fn verdict(geom: &[GeomCell], probes: &[ProbeCell], control_held: bool) -> Verdict {
94    let geom_bad = geom
95        .iter()
96        .any(|c| matches!(c, GeomCell::Mismatch { .. }) || matches!(c, GeomCell::Error { .. }));
97    let probes_bad = probes.iter().any(|c| !matches!(c, ProbeCell::Pass { .. }));
98
99    if !control_held {
100        return Verdict::Vacuous;
101    }
102    if geom_bad || probes_bad {
103        return Verdict::Fail;
104    }
105    Verdict::Pass
106}
107
108/// 2026-09-26: Fold the integrity and concurrency legs into the verdict: a
109/// Pass becomes Fail when an integrity leg failed or the concurrency sweep is
110/// not clean. Fail and Vacuous stay as they are.
111pub fn with_runtime_checks(
112    base: Verdict,
113    integrity_failed: bool,
114    concurrency_clean: bool,
115) -> Verdict {
116    match base {
117        Verdict::Pass if integrity_failed || !concurrency_clean => Verdict::Fail,
118        other => other,
119    }
120}
121
122/// 2026-09-26: How many geometry cells asserted something (`Match` or
123/// `Mismatch`). Reported beside the verdict, because a run where every cell was
124/// `Unmeasured` passes `verdict`.
125pub fn asserted_cells(geom: &[GeomCell]) -> usize {
126    geom.iter()
127        .filter(|c| matches!(c, GeomCell::Match { .. } | GeomCell::Mismatch { .. }))
128        .count()
129}
130
131#[cfg(test)]
132#[path = "score_tests.rs"]
133mod score_tests;
metrale_bench/benchmarks/vision/score.rs

metrale_bench/benchmarks/vision/
score.rs
