metrale_bench/benchmarks/video/
score.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The video benchmark's cells and verdict. A run that asserted
4//! nothing is INCONCLUSIVE; a run whose no-video control named the whole
5//! color sequence is VACUOUS; otherwise it passes only if every asserted cell
6//! passed.
7//!
8//! Owner: bench, video.
9//! Invariants: none beyond the types.
10
11use std::fmt;
12
13/// 2026-09-26: How one ordered-color reading came out.
14#[derive(Debug, Clone, PartialEq, Eq)]
15pub enum OrderCell {
16    /// 2026-09-26: The reply named the clip's colors in the right order.
17    Match {
18        clip: &'static str,
19        seen: String,
20    },
21    /// 2026-09-26: The reply named palette colors, but not this clip's
22    /// sequence.
23    WrongOrder {
24        clip: &'static str,
25        want: String,
26        got: String,
27    },
28    /// 2026-09-26: The reply named no palette color.
29    NotSeen {
30        clip: &'static str,
31        reply: String,
32    },
33    /// 2026-09-26: Skipped: the server cannot decode the clip's container.
34    Skipped {
35        clip: &'static str,
36        why: String,
37    },
38    Error {
39        clip: &'static str,
40        msg: String,
41    },
42}
43
44/// 2026-09-26: How one other leg came out.
45#[derive(Debug, Clone, PartialEq, Eq)]
46pub enum CountCell {
47    Match { id: &'static str, detail: String },
48    Mismatch { id: &'static str, detail: String },
49    Skipped { id: &'static str, why: String },
50    Error { id: &'static str, msg: String },
51}
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54pub enum Verdict {
55    Pass,
56    Fail,
57    /// 2026-09-26: The no-video control named every palette color, so the
58    /// other legs are not evidence.
59    Vacuous,
60    /// 2026-09-26: Nothing was asserted: every leg was skipped.
61    Inconclusive,
62}
63
64impl fmt::Display for Verdict {
65    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
66        f.write_str(match self {
67            Verdict::Pass => "PASS",
68            Verdict::Fail => "FAIL",
69            Verdict::Vacuous => "VACUOUS",
70            Verdict::Inconclusive => "INCONCLUSIVE",
71        })
72    }
73}
74
75/// 2026-09-26: Palette colors found in `reply` as standalone words, each once,
76/// in the order of first appearance, so a recap ("the red one was first") does
77/// not add a color.
78pub fn colors_in_order(reply: &str, palette: &[&str]) -> Vec<String> {
79    let lower = reply.to_lowercase();
80    let mut hits: Vec<(usize, String)> = Vec::new();
81    for c in palette {
82        if let Some(at) = crate::benchmarks::first_standalone_term(&lower, c) {
83            hits.push((at, (*c).to_string()));
84        }
85    }
86    hits.sort_by_key(|(at, _)| *at);
87    hits.into_iter().map(|(_, c)| c).collect()
88}
89
90/// 2026-09-26: Did the reply name exactly this clip's colors, in order?
91pub fn order_matches(reply: &str, want: &[&str], palette: &[&str]) -> bool {
92    let got = colors_in_order(reply, palette);
93    got.len() == want.len() && got.iter().zip(want).all(|(g, w)| g == w)
94}
95
96/// 2026-09-26: Cells that count toward the verdict: everything except
97/// `Skipped`. An `Error` counts as asserted and not passed.
98pub fn asserted(order: &[OrderCell], counts: &[CountCell]) -> usize {
99    order
100        .iter()
101        .filter(|c| {
102            matches!(
103                c,
104                OrderCell::Match { .. }
105                    | OrderCell::WrongOrder { .. }
106                    | OrderCell::NotSeen { .. }
107                    | OrderCell::Error { .. }
108            )
109        })
110        .count()
111        + counts
112            .iter()
113            .filter(|c| {
114                matches!(
115                    c,
116                    CountCell::Match { .. } | CountCell::Mismatch { .. } | CountCell::Error { .. }
117                )
118            })
119            .count()
120}
121
122pub fn passed(order: &[OrderCell], counts: &[CountCell]) -> usize {
123    order
124        .iter()
125        .filter(|c| matches!(c, OrderCell::Match { .. }))
126        .count()
127        + counts
128            .iter()
129            .filter(|c| matches!(c, CountCell::Match { .. }))
130            .count()
131}
132
133/// 2026-09-26: Inconclusive when nothing was asserted, else Vacuous when
134/// `control_held` is false, else Pass only if every asserted cell passed.
135pub fn verdict(order: &[OrderCell], counts: &[CountCell], control_held: bool) -> Verdict {
136    let asserted_n = asserted(order, counts);
137    if asserted_n == 0 {
138        return Verdict::Inconclusive;
139    }
140    if !control_held {
141        return Verdict::Vacuous;
142    }
143    if passed(order, counts) == asserted_n {
144        Verdict::Pass
145    } else {
146        Verdict::Fail
147    }
148}
149
150#[cfg(test)]
151#[path = "score_tests.rs"]
152mod score_tests;
metrale_bench/benchmarks/video/score.rs

metrale_bench/benchmarks/video/
score.rs
