metrale_closure/
layout.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Which files a `(hardware, model, quant)` kernel target is
4//! compiled from.
5//!
6//! The kernels build script (`crates/kernels/build_resolve.rs`) compiles what
7//! it returns, the benchmark gate (`metrale-bench`'s `gate::taxon`) reads
8//! its inputs, and `crates/kernels/tests/kernels_structure.rs` grades the
9//! tree through it. `scripts/lib/kernel_layout.py` mirrors it for the Python
10//! checkers, and `crates/kernels/tests/layout_mirror.rs` holds the two to the
11//! same answer on the real tree.
12//!
13//! # Layers
14//!
15//! A target reads up to four directories, each a layer:
16//!
17//! | role   | tier   | directory                                  |
18//! |--------|--------|--------------------------------------------|
19//! | leaf   | own    | `kernels/<hw>/<source model>/<quant>/`     |
20//! | leaf   | parent | `kernels/<parent>/<source model>/<quant>/` |
21//! | common | own    | `kernels/<hw>/common/`                     |
22//! | common | parent | `kernels/<parent>/common/`                 |
23//!
24//! `<parent>` is `[hardware] inherits` in the tree's `HARDWARE.toml`; a tree
25//! that inherits nothing has two layers. `<source model>` is the model itself
26//! unless its `MODEL.toml` redirects with `[model] kernel_source`. A layer's
27//! entries are the kernel sources and headers its directory holds plus what
28//! its `KERNEL.toml` `[sources] use` brings in.
29//!
30//! Within a role the own tier wins by file name; the leaf role wins over the
31//! common role by source file stem. Every such win must be declared in
32//! `[shadow]` of the winner's `KERNEL.toml`, with a reason: an undeclared one
33//! is an error, and so is a declaration with nothing to shadow.
34//!
35//! Module identity is the file stem ([`Layout::modules`]), whichever layer
36//! the file came from.
37//!
38//! # Materialisation
39//!
40//! Entries are compiled in their role directory, not where the file lives:
41//! the compiler resolves a quoted `#include` against the including file's
42//! directory, so a `use`d source must see the headers of the layer that uses
43//! it. The build script stages each role into `OUT_DIR` before compiling
44//! (`crates/kernels/build_stage.rs`); [`Layout::leaf`] and [`Layout::common`]
45//! are those two directories' contents.
46//!
47//! Owner: metrale-closure (kernel layout).
48//! Invariants:
49//! - In a `Layout` that [`discover`] returns, every win of one layer over
50//!   another is in `shadows` with its declared reason, and every `[shadow]`
51//!   key names a win.
52//! - No entry's file is a symlink.
53
54use std::collections::{BTreeMap, BTreeSet};
55use std::path::{Path, PathBuf};
56
57pub use crate::layout_manifest::{
58    HEADER_EXTS, Hardware, KernelManifest, LayoutError, hardware, kernel_manifest,
59    kernel_source_dir, source_ext, stem_of,
60};
61use crate::layout_scan::{has_ext, layer, layer_contents};
62
63/// 2026-09-26: One compiled unit: `(hardware, model, quant)`.
64#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
65pub struct Target {
66    pub hardware: String,
67    pub model: String,
68    pub quant: String,
69}
70
71impl std::fmt::Display for Target {
72    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
73        write!(f, "{}/{}/{}", self.hardware, self.model, self.quant)
74    }
75}
76
77#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
78pub enum Role {
79    Leaf,
80    Common,
81}
82
83#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
84pub enum Tier {
85    Own,
86    Parent,
87}
88
89/// 2026-09-26: One of a target's directories, whether or not it exists.
90#[derive(Debug, Clone, PartialEq, Eq)]
91pub struct Layer {
92    pub role: Role,
93    pub tier: Tier,
94    /// 2026-09-26: The `kernels/<hardware>` tree this directory is under,
95    /// which its `[sources] use` paths are relative to.
96    pub hardware: String,
97    pub dir: PathBuf,
98    pub manifest: Option<KernelManifest>,
99}
100
101/// 2026-09-26: One file a target compiles or stages beside what it compiles.
102#[derive(Debug, Clone, PartialEq, Eq)]
103pub struct Entry {
104    /// 2026-09-26: File name in its role directory.
105    pub name: String,
106    /// 2026-09-26: The file it resolves to.
107    pub source: PathBuf,
108    /// 2026-09-26: Index into [`Layout::layers`].
109    pub layer: usize,
110    /// 2026-09-26: Brought in by `[sources] use` rather than held by the
111    /// directory.
112    pub used: bool,
113}
114
115/// 2026-09-26: A declared, verified shadow.
116#[derive(Debug, Clone, PartialEq, Eq)]
117pub struct Shadow {
118    pub role: Role,
119    pub name: String,
120    pub winner: PathBuf,
121    pub loser: PathBuf,
122    pub reason: String,
123}
124
125#[derive(Debug, Clone)]
126pub struct Layout {
127    pub target: Target,
128    pub hardware: Hardware,
129    /// 2026-09-26: Own leaf, parent leaf, own common, parent common; the
130    /// parent pair only when the tree inherits.
131    pub layers: Vec<Layer>,
132    /// 2026-09-26: Leaf-role files by name.
133    pub leaf: BTreeMap<String, Entry>,
134    /// 2026-09-26: Common-role files by name.
135    pub common: BTreeMap<String, Entry>,
136    /// 2026-09-26: Subdirectories of the role directories (e.g. vendored
137    /// headers), by name.
138    pub leaf_subdirs: BTreeMap<String, PathBuf>,
139    pub common_subdirs: BTreeMap<String, PathBuf>,
140    pub shadows: Vec<Shadow>,
141    /// 2026-09-26: The model whose own-tier leaf directory this target
142    /// reads, after redirect.
143    pub source_model: String,
144    /// 2026-09-26: `kernels/<hw>/<model>`, the target's own model directory.
145    pub model_dir: PathBuf,
146}
147
148impl Layout {
149    /// 2026-09-26: The `KERNEL.toml`s that exist, least specific first:
150    /// parent common, own common, parent leaf, own leaf. The build script
151    /// merges them in this order (`build_resolve.rs`).
152    pub fn configs(&self) -> Vec<PathBuf> {
153        let mut order: Vec<&Layer> = self.layers.iter().collect();
154        order.sort_by_key(|l| (std::cmp::Reverse(l.role), std::cmp::Reverse(l.tier)));
155        order
156            .into_iter()
157            .filter_map(|l| l.manifest.as_ref().map(|m| m.path.clone()))
158            .collect()
159    }
160
161    /// 2026-09-26: `(stem, entry)` for every module source, sorted by stem:
162    /// leaf over common.
163    pub fn modules(&self) -> Vec<(String, &Entry)> {
164        let ext = self.hardware.source_ext;
165        let mut out: BTreeMap<String, &Entry> = BTreeMap::new();
166        for map in [&self.common, &self.leaf] {
167            for (name, entry) in map {
168                if has_ext(name, ext) {
169                    out.insert(stem_of(name).to_string(), entry);
170                }
171            }
172        }
173        out.into_iter().collect()
174    }
175
176    /// 2026-09-26: Module sources, sorted.
177    pub fn sources(&self) -> Vec<PathBuf> {
178        let mut out: Vec<PathBuf> = self
179            .modules()
180            .into_iter()
181            .map(|(_, e)| e.source.clone())
182            .collect();
183        out.sort();
184        out
185    }
186
187    /// 2026-09-26: Every file the target's compile reads through this
188    /// resolution: the role entries, every file under the role
189    /// subdirectories, and the `KERNEL.toml`s. The build script watches them
190    /// for rebuilds (`build_plan.rs`).
191    pub fn inputs(&self) -> BTreeSet<PathBuf> {
192        let mut out: BTreeSet<PathBuf> = self
193            .leaf
194            .values()
195            .chain(self.common.values())
196            .map(|e| e.source.clone())
197            .collect();
198        for dir in self
199            .leaf_subdirs
200            .values()
201            .chain(self.common_subdirs.values())
202        {
203            files_under(dir, &mut out);
204        }
205        out.extend(self.configs());
206        out
207    }
208
209    pub fn role(&self, role: Role) -> (&BTreeMap<String, Entry>, &BTreeMap<String, PathBuf>) {
210        match role {
211            Role::Leaf => (&self.leaf, &self.leaf_subdirs),
212            Role::Common => (&self.common, &self.common_subdirs),
213        }
214    }
215}
216
217fn files_under(dir: &Path, out: &mut BTreeSet<PathBuf>) {
218    let Ok(entries) = std::fs::read_dir(dir) else {
219        return;
220    };
221    for entry in entries.flatten() {
222        let path = entry.path();
223        if path.is_dir() {
224            files_under(&path, out);
225        } else {
226            out.insert(path);
227        }
228    }
229}
230
231/// 2026-09-26: Every symlink under `kernels/`, sorted. The structure test
232/// (`kernels_structure.rs`) refuses any.
233pub fn symlinks(root: &Path) -> Vec<PathBuf> {
234    let mut out = Vec::new();
235    fn walk_dir(dir: &Path, out: &mut Vec<PathBuf>) {
236        let Ok(entries) = std::fs::read_dir(dir) else {
237            return;
238        };
239        for entry in entries.flatten() {
240            let path = entry.path();
241            if path.is_symlink() {
242                out.push(path);
243            } else if path.is_dir() {
244                walk_dir(&path, out);
245            }
246        }
247    }
248    walk_dir(&root.join("kernels"), &mut out);
249    out.sort();
250    out
251}
252
253fn subdirs(dir: &Path) -> Vec<String> {
254    let Ok(entries) = std::fs::read_dir(dir) else {
255        return Vec::new();
256    };
257    let mut out: Vec<String> = entries
258        .flatten()
259        .filter(|e| e.path().is_dir())
260        .filter_map(|e| e.file_name().to_str().map(str::to_string))
261        .collect();
262    out.sort();
263    out
264}
265
266/// 2026-09-26: The leaf model directory of `model` in the tree `hw_dir`,
267/// after that tree's own redirect, or where it would be when the tree has no
268/// such model.
269fn leaf_model_dir(hw_dir: &Path, model: &str) -> Result<PathBuf, LayoutError> {
270    let dir = hw_dir.join(model);
271    if dir.join("MODEL.toml").is_file() {
272        kernel_source_dir(&dir)
273    } else {
274        Ok(dir)
275    }
276}
277
278/// 2026-09-26: Every target in the tree: each `kernels/<hw>` with a
279/// `HARDWARE.toml`, each model directory under it with a `MODEL.toml`, and
280/// every quant directory its source model owns in the tree or in the tree it
281/// inherits.
282pub fn walk(root: &Path) -> Result<Vec<Target>, LayoutError> {
283    let kernels = root.join("kernels");
284    let mut out = Vec::new();
285    for hw in subdirs(&kernels) {
286        let hw_dir = kernels.join(&hw);
287        if !hw_dir.join("HARDWARE.toml").is_file() {
288            continue;
289        }
290        let hardware = hardware(&kernels, &hw)?;
291        for model in subdirs(&hw_dir) {
292            let model_dir = hw_dir.join(&model);
293            if !model_dir.join("MODEL.toml").is_file() {
294                continue;
295            }
296            let own_src = kernel_source_dir(&model_dir)?;
297            let mut quants: BTreeSet<String> = subdirs(&own_src).into_iter().collect();
298            if let Some(parent) = &hardware.inherits {
299                let src_model = own_src
300                    .file_name()
301                    .and_then(|n| n.to_str())
302                    .unwrap_or(&model);
303                quants.extend(subdirs(&leaf_model_dir(&kernels.join(parent), src_model)?));
304            }
305            for quant in quants {
306                out.push(Target {
307                    hardware: hw.clone(),
308                    model: model.clone(),
309                    quant,
310                });
311            }
312        }
313    }
314    Ok(out)
315}
316
317/// 2026-09-26: Resolve one target.
318pub fn discover(root: &Path, target: &Target) -> Result<Layout, LayoutError> {
319    let kernels = root.join("kernels");
320    let hardware = hardware(&kernels, &target.hardware)?;
321    let hw_dir = kernels.join(&target.hardware);
322    let model_dir = hw_dir.join(&target.model);
323    if !model_dir.is_dir() {
324        return Err(LayoutError::UnknownModel(model_dir));
325    }
326    let own_src = kernel_source_dir(&model_dir)?;
327    let source_model = own_src
328        .file_name()
329        .and_then(|n| n.to_str())
330        .unwrap_or(&target.model)
331        .to_string();
332
333    let mut layers = vec![layer(
334        Role::Leaf,
335        Tier::Own,
336        &target.hardware,
337        own_src.join(&target.quant),
338    )?];
339    if let Some(parent) = &hardware.inherits {
340        let parent_dir = kernels.join(parent);
341        let parent_src = leaf_model_dir(&parent_dir, &source_model)?;
342        layers.push(layer(
343            Role::Leaf,
344            Tier::Parent,
345            parent,
346            parent_src.join(&target.quant),
347        )?);
348    }
349    layers.push(layer(
350        Role::Common,
351        Tier::Own,
352        &target.hardware,
353        hw_dir.join("common"),
354    )?);
355    if let Some(parent) = &hardware.inherits {
356        layers.push(layer(
357            Role::Common,
358            Tier::Parent,
359            parent,
360            kernels.join(parent).join("common"),
361        )?);
362    }
363    if !layers.iter().any(|l| l.dir.is_dir()) {
364        return Err(LayoutError::NoKernelDirectory(target.to_string()));
365    }
366
367    let mut leaf = BTreeMap::new();
368    let mut common = BTreeMap::new();
369    let mut leaf_subdirs = BTreeMap::new();
370    let mut common_subdirs = BTreeMap::new();
371    let mut shadows = Vec::new();
372    let mut declared_used: BTreeSet<(usize, String)> = BTreeSet::new();
373    for (idx, l) in layers.iter().enumerate() {
374        let (entries, dirs) = match l.role {
375            Role::Leaf => (&mut leaf, &mut leaf_subdirs),
376            Role::Common => (&mut common, &mut common_subdirs),
377        };
378        let (own_entries, own_dirs) = layer_contents(&kernels, l, idx, hardware.source_ext)?;
379        for (name, entry) in own_entries {
380            match entries.get(&name) {
381                None => {
382                    entries.insert(name, entry);
383                }
384                // 2026-09-26: A lower tier's namesake: the earlier (own)
385                // entry wins, and that win is a shadow the winner must have
386                // declared.
387                Some(winner) => {
388                    let winner = winner.clone();
389                    let shadow = declare(&layers, &winner, &entry.source, &mut declared_used)?;
390                    shadows.push(shadow);
391                }
392            }
393        }
394        for (name, dir) in own_dirs {
395            dirs.entry(name).or_insert(dir);
396        }
397    }
398    // 2026-09-26: Leaf over common, by stem, sources only.
399    for (name, entry) in &leaf {
400        if !has_ext(name, hardware.source_ext) {
401            continue;
402        }
403        let stem = stem_of(name);
404        let Some((_, loser)) = common
405            .iter()
406            .find(|(n, _)| has_ext(n, hardware.source_ext) && stem_of(n) == stem)
407        else {
408            continue;
409        };
410        shadows.push(declare(&layers, entry, &loser.source, &mut declared_used)?);
411    }
412    for (idx, l) in layers.iter().enumerate() {
413        if let Some(m) = &l.manifest {
414            for stem in m.shadow.keys() {
415                if !declared_used.contains(&(idx, stem.clone())) {
416                    return Err(LayoutError::DeadShadow {
417                        manifest: m.path.clone(),
418                        stem: stem.clone(),
419                    });
420                }
421            }
422        }
423    }
424    shadows.sort_by(|a, b| (a.role, &a.name, &a.loser).cmp(&(b.role, &b.name, &b.loser)));
425    Ok(Layout {
426        target: target.clone(),
427        hardware,
428        layers,
429        leaf,
430        common,
431        leaf_subdirs,
432        common_subdirs,
433        shadows,
434        source_model,
435        model_dir,
436    })
437}
438
439/// 2026-09-26: A winner over `loser` is a shadow the winner's manifest must
440/// declare.
441fn declare(
442    layers: &[Layer],
443    winner: &Entry,
444    loser: &Path,
445    used: &mut BTreeSet<(usize, String)>,
446) -> Result<Shadow, LayoutError> {
447    if winner.source == loser {
448        return Err(LayoutError::SelfShadow {
449            name: winner.name.clone(),
450            source: winner.source.clone(),
451        });
452    }
453    let l = &layers[winner.layer];
454    let stem = stem_of(&winner.name).to_string();
455    let manifest_path = l.dir.join("KERNEL.toml");
456    let reason = l
457        .manifest
458        .as_ref()
459        .and_then(|m| m.shadow.get(&stem))
460        .ok_or_else(|| LayoutError::UndeclaredShadow {
461            name: winner.name.clone(),
462            winner: winner.source.clone(),
463            loser: loser.to_path_buf(),
464            manifest: manifest_path,
465        })?;
466    used.insert((winner.layer, stem));
467    Ok(Shadow {
468        role: l.role,
469        name: winner.name.clone(),
470        winner: winner.source.clone(),
471        loser: loser.to_path_buf(),
472        reason: reason.clone(),
473    })
474}
475
476#[cfg(test)]
477#[path = "layout_tests.rs"]
478mod layout_tests;
metrale_closure/layout.rs

metrale_closure/
layout.rs
