metrale_model_layers/
vision_item.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-25: The unit of vision input handed to the model.
4//!
5//! Owner: model-layers (vision input).
6//! Invariants: none beyond the types.
7
8/// 2026-09-25: One vision item: a still image, or a video, ready for the encoder.
9///
10/// `groups` holds the temporal groups: one for a still image (`image`), and
11/// `grid_t` for a video, each group a full `grid_h × grid_w` patch plane built
12/// from `temporal_patch_size` consecutive frames. All groups share the item's
13/// one grid, so the encoder takes each group as an image of that grid.
14///
15/// The group count lives in the item rather than in a parallel vector, so the
16/// pad run (`pad_count`) and the encoder grids both read it from `t_len`.
17#[derive(Debug, Clone, PartialEq)]
18pub struct VisionItem {
19    /// 2026-09-25: Per temporal group: `[grid_h * grid_w, C * temporal_patch_size * patch^2]`.
20    pub groups: Vec<Vec<f32>>,
21    /// 2026-09-25: Pre-merge patch grid, shared by all of this item's groups.
22    pub grid_h: usize,
23    pub grid_w: usize,
24}
25
26impl VisionItem {
27    /// 2026-09-25: A still image: one temporal group.
28    pub fn image(pixels: Vec<f32>, grid_h: usize, grid_w: usize) -> Self {
29        Self {
30            groups: vec![pixels],
31            grid_h,
32            grid_w,
33        }
34    }
35
36    /// 2026-09-25: Temporal extent, in groups; at least 1.
37    pub fn t_len(&self) -> usize {
38        self.groups.len().max(1)
39    }
40
41    /// 2026-09-25: Merged tokens this item occupies in the prompt: `t_len` times
42    /// the spatially merged grid. The chat request builder uses it as the
43    /// length of the item's pad run.
44    pub fn pad_count(&self, spatial_merge_size: usize) -> usize {
45        let sms = spatial_merge_size.max(1);
46        self.t_len() * (self.grid_h / sms) * (self.grid_w / sms)
47    }
48}
metrale_model_layers/vision_item.rs

metrale_model_layers/
vision_item.rs
