metrale_bench/gate/
mod.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The pull-request gate: benchmark records committed beside the code they
4//! measured.
5//!
6//! Owner: bench gate.
7//! Invariants: none beyond the types.
8//!
9//! Each run writes one record file under `.benchmarks/<id>/`; a default-variant record
10//! is named `YYYY-MM-DD-<sha>.json` ([`record_path()`]). The thresholds a record must meet
11//! are assembled from every `kernels/<hw>/<model>/BENCH.toml` ([`read_baseline`]).
12//! [`check_gates`] judges every id in [`REQUIRED_GATES`] against the committed files.
13
14pub mod amnesty;
15pub mod bench;
16pub mod card;
17pub mod check;
18mod check_fmt;
19mod check_group;
20pub use check_group::shards_owed;
21pub mod check_paths;
22pub mod closure;
23pub mod codeowners;
24pub mod coverage;
25pub mod hermetic;
26pub mod recipe_closure;
27pub mod record;
28mod record_env;
29mod record_path;
30pub use record_path::shard_suffix;
31pub mod record_serve;
32mod record_summary;
33mod record_write;
34pub mod scoring;
35pub mod signing;
36pub mod taxon;
37
38pub mod pr_taxonomy;
39pub mod required;
40pub mod telemetry;
41
42use std::path::{Path, PathBuf};
43use std::time::Duration;
44
45use anyhow::{Context, Result, bail};
46
47pub use check::{
48    Comparison, GateStatus, Standing, check_gates, check_record, compare, record_covers,
49    record_standing, records_newest_first,
50};
51pub use record::{
52    Bound, GateBaseline, GateRecord, HardwareBaseline, ModelBaseline, date_of,
53    merge_serve_overrides, now_secs, read_baseline, read_record, record_path, record_path_for,
54    variant_slug, write_record,
55};
56
57/// 2026-09-26: The ids of `coverage::REQUIRED`, in its order: the gates [`check_gates`]
58/// judges. A test in `registry.rs` looks up each id as a registered benchmark.
59pub const REQUIRED_GATES: [&str; 17] = [
60    coverage::REQUIRED[0].id,
61    coverage::REQUIRED[1].id,
62    coverage::REQUIRED[2].id,
63    coverage::REQUIRED[3].id,
64    coverage::REQUIRED[4].id,
65    coverage::REQUIRED[5].id,
66    coverage::REQUIRED[6].id,
67    coverage::REQUIRED[7].id,
68    coverage::REQUIRED[8].id,
69    coverage::REQUIRED[9].id,
70    coverage::REQUIRED[10].id,
71    coverage::REQUIRED[11].id,
72    coverage::REQUIRED[12].id,
73    coverage::REQUIRED[13].id,
74    coverage::REQUIRED[14].id,
75    coverage::REQUIRED[15].id,
76    coverage::REQUIRED[16].id,
77];
78
79/// 2026-09-26: The timeout for the endpoint's `/hardware` fetch when a gate record is
80/// written (`bench_record.rs` in `metrale-server`).
81pub const HARDWARE_TIMEOUT: Duration = Duration::from_secs(10);
82
83/// 2026-09-26: The paths a diff must touch to invalidate a gate record, before the
84/// per-gate exclusions in `coverage` subtract from them ([`check::record_covers`]).
85pub use coverage::PERF_PATHS;
86
87/// 2026-09-26: `.benchmarks/<benchmark_id>` under `root`.
88pub fn gate_dir(root: &Path, benchmark_id: &str) -> PathBuf {
89    root.join(".benchmarks").join(benchmark_id)
90}
91
92/// 2026-09-26: The full 40-hex commit id `rev` resolves to in this working tree.
93///
94/// # Errors
95/// If git cannot run, `rev` does not name a commit, or git prints something other
96/// than 40 hex digits.
97pub fn git_rev_parse(root: &Path, rev: &str) -> Result<String> {
98    let out = std::process::Command::new("git")
99        .arg("-C")
100        .arg(root)
101        .args(["rev-parse", "--verify", &format!("{rev}^{{commit}}")])
102        .stdin(std::process::Stdio::null())
103        .output()
104        .context("running git rev-parse")?;
105    if !out.status.success() {
106        bail!(
107            "{rev:?} does not name a commit in {}: {}",
108            root.display(),
109            String::from_utf8_lossy(&out.stderr).trim()
110        );
111    }
112    let sha = String::from_utf8_lossy(&out.stdout).trim().to_string();
113    if sha.len() != 40 || !sha.chars().all(|c| c.is_ascii_hexdigit()) {
114        bail!("git rev-parse returned {sha:?}, not a 40-hex commit");
115    }
116    Ok(sha)
117}
118
119/// 2026-09-26: The repository root that contains `dir`, from git itself.
120///
121/// # Errors
122/// If git cannot run, or `dir` is not inside a git working tree.
123pub fn git_rev_parse_toplevel(dir: &Path) -> Result<PathBuf> {
124    let out = std::process::Command::new("git")
125        .arg("-C")
126        .arg(dir)
127        .args(["rev-parse", "--show-toplevel"])
128        .stdin(std::process::Stdio::null())
129        .output()
130        .context("running git rev-parse --show-toplevel")?;
131    if !out.status.success() {
132        bail!(
133            "{} is not inside a git working tree: {}",
134            dir.display(),
135            String::from_utf8_lossy(&out.stderr).trim()
136        );
137    }
138    Ok(PathBuf::from(String::from_utf8_lossy(&out.stdout).trim()))
139}
140
141/// 2026-09-26: The short commit id of `HEAD` (`--short=10`), or the trimmed value of
142/// `METRALE_GATE_SHA` when that is set, for a checkout without git metadata.
143///
144/// # Errors
145/// If `METRALE_GATE_SHA` is set but empty, or git fails or prints nothing.
146pub fn git_sha(root: &Path) -> Result<String> {
147    if let Some(explicit) = std::env::var_os("METRALE_GATE_SHA") {
148        let sha = explicit.to_string_lossy().trim().to_string();
149        if sha.is_empty() {
150            bail!("METRALE_GATE_SHA is set but empty");
151        }
152        return Ok(sha);
153    }
154    let out = std::process::Command::new("git")
155        .arg("-C")
156        .arg(root)
157        .args(["rev-parse", "--short=10", "HEAD"])
158        .stdin(std::process::Stdio::null())
159        .output()
160        .context("running git rev-parse")?;
161    if !out.status.success() {
162        bail!(
163            "git rev-parse failed — {} is not a git checkout (or git is \
164             missing); set METRALE_GATE_SHA to record a gate run",
165            root.display()
166        );
167    }
168    let sha = String::from_utf8_lossy(&out.stdout).trim().to_string();
169    if sha.is_empty() {
170        bail!("git rev-parse printed nothing");
171    }
172    Ok(sha)
173}
174
175/// 2026-09-26: The files under [`PERF_PATHS`] that `git status` reports as modified,
176/// staged or untracked in this working tree, sorted and deduplicated. The gate run
177/// records this list, since a dirty perf path means the binary was not built from the
178/// commit the record names.
179///
180/// Untracked files are listed one by one (`--untracked-files=all`), not collapsed to
181/// their directory. `.benchmarks` is not a perf path, so a record file written by an
182/// earlier gate run is not reported.
183///
184/// # Errors
185/// If git cannot run or `git status` fails, rather than reporting a clean tree.
186pub fn dirty_perf_paths(root: &Path) -> Result<Vec<String>> {
187    let out = std::process::Command::new("git")
188        .arg("-C")
189        .arg(root)
190        .args(["status", "--porcelain", "--untracked-files=all", "--"])
191        .args(PERF_PATHS)
192        .stdin(std::process::Stdio::null())
193        .output()
194        .context("running git status")?;
195    if !out.status.success() {
196        bail!(
197            "git status failed in {} — cannot tell whether the measured binary \
198             matches the commit being stamped",
199            root.display()
200        );
201    }
202    let mut paths: Vec<String> = String::from_utf8_lossy(&out.stdout)
203        .lines()
204        .filter_map(|line| line.get(3..))
205        // 2026-09-26: A rename reads `R  old -> new`; keep the destination, the path
206        // that exists in the tree now.
207        .map(|entry| match entry.split_once(" -> ") {
208            Some((_, dest)) => dest.trim().to_string(),
209            None => entry.trim().to_string(),
210        })
211        .filter(|p| !p.is_empty())
212        .collect();
213    paths.sort();
214    paths.dedup();
215    Ok(paths)
216}
217
218#[cfg(test)]
219#[path = "tests.rs"]
220mod tests;
221
222#[cfg(test)]
223#[path = "record_contract_tests.rs"]
224mod record_contract_tests;
225
226#[cfg(test)]
227#[path = "energy_ceiling_tests.rs"]
228mod energy_ceiling_tests;
229
230#[cfg(test)]
231#[path = "variant_tests.rs"]
232mod variant_tests;
233
234#[cfg(test)]
235#[path = "fixture_baseline.rs"]
236mod fixture_baseline;
237
238#[cfg(test)]
239#[path = "coverage_map_tests.rs"]
240mod coverage_map_tests;
241
242#[cfg(test)]
243#[path = "test_only_coverage_tests.rs"]
244mod test_only_coverage_tests;
245
246#[cfg(test)]
247#[path = "coverage_promotion_tests.rs"]
248mod coverage_promotion_tests;
249#[cfg(test)]
250#[path = "coverage_tests.rs"]
251mod coverage_tests;
252
253#[cfg(test)]
254#[path = "coverage_squash_tests.rs"]
255mod coverage_squash_tests;
256
257#[cfg(test)]
258#[path = "hardening_tests.rs"]
259mod hardening_tests;
260
261#[cfg(test)]
262#[path = "card_tests.rs"]
263mod card_tests;
264
265pub mod agreement;
266pub mod group;
267
268#[cfg(test)]
269#[path = "group_tests.rs"]
270mod group_tests;
271
272#[cfg(test)]
273#[path = "group_verdict_tests.rs"]
274mod group_verdict_tests;
275
276#[cfg(test)]
277#[path = "agreement_tests.rs"]
278mod agreement_tests;
279
280#[cfg(test)]
281#[path = "signer_registry_tests.rs"]
282mod signer_registry_tests;
283
284#[cfg(test)]
285#[path = "signing_tests.rs"]
286mod signing_tests;
287
288#[cfg(test)]
289#[path = "amnesty_tests.rs"]
290mod amnesty_tests;
291#[cfg(test)]
292#[path = "recipe_closure_tests.rs"]
293mod recipe_closure_tests;
294#[cfg(test)]
295#[path = "standing_tests.rs"]
296mod standing_tests;
297
298#[cfg(test)]
299#[path = "dirty_tests.rs"]
300mod dirty_tests;
301
302#[cfg(test)]
303#[path = "override_tests.rs"]
304mod override_tests;
metrale_bench/gate/mod.rs

metrale_bench/gate/
mod.rs
