metrale_bench/benchmarks/vision/
provision.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The vision fixtures, written to `~/.metrale/artifacts/vision` on
4//! `load()`.
5//!
6//! The images are compiled into the binary from `crates/bench/assets/vision/`
7//! and written out only when a file's bytes differ (`write_asset_bytes`),
8//! behind a content-derived [`Stamp`], so a run that already provisioned
9//! writes nothing. The written directory lets an operator send the exact bytes
10//! a run used with any HTTP client.
11//!
12//! The same files are in `tests/fixtures/images/`, and
13//! `assets_match_the_repo_fixtures` fails when the two copies differ. They are
14//! copied into the crate because `cargo package` does not include a path
15//! outside it. `scripts/gen_test_images.py` generates `01` to `09`; it does
16//! not generate `10` to `16`.
17//!
18//! Owner: bench, vision.
19//! Invariants:
20//! - `provision` commits the stamp only after every fixture write succeeded.
21
22use anyhow::Result;
23
24use crate::artifacts::{ArtifactStore, Stamp, write_asset_bytes};
25
26/// 2026-09-26: The geometry ladder as `(name, bytes, width, height)`: `01` to
27/// `08` square, landscape and portrait sizes up to 1280x720, `09` over the
28/// 1280 px fallback clamp, `10` and `11` geometry extremes, and `12` to `14`
29/// decode variants at 224x224.
30pub const FIXTURES: &[(&str, &[u8], u32, u32)] = &[
31    (
32        "01_square_224.png",
33        include_bytes!("../../../assets/vision/01_square_224.png"),
34        224,
35        224,
36    ),
37    (
38        "02_square_336.png",
39        include_bytes!("../../../assets/vision/02_square_336.png"),
40        336,
41        336,
42    ),
43    (
44        "03_landscape_512x384.png",
45        include_bytes!("../../../assets/vision/03_landscape_512x384.png"),
46        512,
47        384,
48    ),
49    (
50        "04_wide_640x360.png",
51        include_bytes!("../../../assets/vision/04_wide_640x360.png"),
52        640,
53        360,
54    ),
55    (
56        "05_square_768.png",
57        include_bytes!("../../../assets/vision/05_square_768.png"),
58        768,
59        768,
60    ),
61    (
62        "06_wide_1024x576.png",
63        include_bytes!("../../../assets/vision/06_wide_1024x576.png"),
64        1024,
65        576,
66    ),
67    (
68        "07_hd_1280x720.png",
69        include_bytes!("../../../assets/vision/07_hd_1280x720.png"),
70        1280,
71        720,
72    ),
73    (
74        "08_portrait_480x854.png",
75        include_bytes!("../../../assets/vision/08_portrait_480x854.png"),
76        480,
77        854,
78    ),
79    // 2026-09-26: Every fixture above has a long side of at most 1280 px, so
80    // its count is the same whether the engine honours the checkpoint's area
81    // bound or falls back to the 1280 px clamp. 1600x900 is 1.44M px, under the
82    // 16777216 px of the Qwen3.6 preprocessor configs, so at native size it is
83    // 1400 tokens; under the clamp it is served at 1280x736, 920 tokens.
84    (
85        "09_over_clamp_1600x900.png",
86        include_bytes!("../../../assets/vision/09_over_clamp_1600x900.png"),
87        1600,
88        900,
89    ),
90    // 2026-09-26: 8x8 snaps up to one 32 px grid unit, one merged token: the
91    // smallest target the preprocessor produces (`.max(1)` in
92    // `target_size_for`).
93    (
94        "10_tiny_8x8.png",
95        include_bytes!("../../../assets/vision/10_tiny_8x8.png"),
96        8,
97        8,
98    ),
99    // 2026-09-26: 64x2048 has a small area and a long side over the 1280 px
100    // fallback clamp: 128 tokens at native size, 40 under the clamp (served at
101    // 32x1280).
102    (
103        "11_strip_64x2048.png",
104        include_bytes!("../../../assets/vision/11_strip_64x2048.png"),
105        64,
106        2048,
107    ),
108    // 2026-09-26: Decode variants at 224x224, so each asserts the same 49
109    // tokens as `01`; only the pixel format differs. `12` is RGBA with a
110    // varying alpha channel, which the engine's `to_rgb8()` drops.
111    (
112        "12_rgba_224.png",
113        include_bytes!("../../../assets/vision/12_rgba_224.png"),
114        224,
115        224,
116    ),
117    // 2026-09-26: Single-channel (grayscale) JPEG.
118    (
119        "13_gray_224.jpg",
120        include_bytes!("../../../assets/vision/13_gray_224.jpg"),
121        224,
122        224,
123    ),
124    // 2026-09-26: 16-bit grayscale PNG.
125    (
126        "14_png16_224.png",
127        include_bytes!("../../../assets/vision/14_png16_224.png"),
128        224,
129        224,
130    ),
131];
132
133/// 2026-09-26: The EXIF-orientation pair, kept out of the geometry ladder
134/// because the integrity leg checks what the model reports, not a token count.
135///
136/// Both store the same pixels, red on the top half and blue on the bottom, at
137/// 224x224. `15` carries `Orientation = 6` ("rotate 90 CW to display"); `16`
138/// carries no EXIF. The engine applies the tag
139/// (`model-layers/src/vision_preprocess.rs`), which carries the stored top
140/// edge to the right, so the tagged image must read "right" and the untagged
141/// one "top".
142pub const EXIF_PAIR: &[(&str, &[u8])] = &[
143    (
144        "15_exif_rot90_224.jpg",
145        include_bytes!("../../../assets/vision/15_exif_rot90_224.jpg"),
146    ),
147    (
148        "16_exif_none_224.jpg",
149        include_bytes!("../../../assets/vision/16_exif_none_224.jpg"),
150    ),
151];
152
153/// 2026-09-26: Content-derived over every fixture's name and bytes, so a
154/// changed image re-provisions without a version bump.
155fn stamp_value() -> String {
156    crate::benchmarks::content_stamp(
157        "vision-fixtures-v1",
158        FIXTURES
159            .iter()
160            .map(|(name, bytes, _, _)| (*name, *bytes))
161            .chain(EXIF_PAIR.iter().copied()),
162    )
163}
164
165/// 2026-09-26: The artifact directory name: `~/.metrale/artifacts/vision`.
166pub const PLUGIN_ID: &str = "vision";
167
168/// 2026-09-26: Write the fixtures. With a current stamp it writes nothing.
169pub fn provision(store: &ArtifactStore) -> Result<std::path::PathBuf> {
170    let dir = store.plugin_dir(PLUGIN_ID)?;
171    let stamp = Stamp::new(&dir, ".provisioned", stamp_value());
172    if stamp.is_current() {
173        return Ok(dir);
174    }
175    for (name, bytes, _, _) in FIXTURES {
176        write_asset_bytes(&dir, name, bytes)?;
177    }
178    for (name, bytes) in EXIF_PAIR {
179        write_asset_bytes(&dir, name, bytes)?;
180    }
181    // 2026-09-26: Last: a stamp written before the writes complete would mark
182    // a partial directory as current.
183    stamp.commit()?;
184    Ok(dir)
185}
186
187#[cfg(test)]
188#[path = "provision_tests.rs"]
189mod provision_tests;
metrale_bench/benchmarks/vision/provision.rs

metrale_bench/benchmarks/vision/
provision.rs
