metrale_bench/benchmarks/vision/
mod.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The `vision-fidelity` benchmark: does the served model see the
4//! image it was sent, at the resolution the serve permits?
5//!
6//! - Geometry: the exact vision-token count of each ladder fixture
7//!   (`provision::FIXTURES`), read from `usage.prompt_tokens` minus a
8//!   calibrated template overhead and compared with `geometry.rs`. A
9//!   preprocessing change that still leaves a recognisable picture moves this
10//!   count, where a capability probe can still pass.
11//! - Capability: probes on the fixtures (`probes.rs`), with a no-image control;
12//!   if the control answers, the run is VACUOUS.
13//! - Integrity and concurrency legs (`driver.rs`).
14//!
15//! `tests/vision_sweep.py` (a keyword rubric on one photo) and
16//! `tests/vit_reference_check.py` (a layer-by-layer diff against an HF
17//! reference) are separate harnesses and assert no vision-token counts.
18//!
19//! Registered in `registry.rs`; model targets gate on it through
20//! `gate = "vision-fidelity"` entries in their BENCH.toml.
21//!
22//! Owner: bench, vision.
23//! Invariants: none beyond the types.
24
25pub mod driver;
26pub mod geometry;
27pub mod probes;
28pub mod provision;
29pub mod request;
30pub mod score;
31
32pub use driver::{DESCRIPTOR, METADATA};
metrale_bench/benchmarks/vision/mod.rs

metrale_bench/benchmarks/vision/
mod.rs
