metrale_bench/gate/
check_group.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: `check_one`'s path for a benchmark group: one verdict from its shard records.
4//!
5//! Owner: bench gate.
6//! Invariants:
7//! - A group passes only on a complete partition of shard records at one commit
8//!   ([`super::group::select_partition`]) whose every shard is for the group and its subject,
9//!   still stands, and passes `shard_problems`, with no transport errors, and whose
10//!   aggregate passes `check_record`.
11//! - A whole-draw record (no shard identity) never counts.
12
13use std::collections::BTreeMap;
14use std::path::{Path, PathBuf};
15
16use super::check::{
17    GateStatus, record_is_for, record_is_required_subject, record_still_stands,
18    records_newest_first,
19};
20use super::group::ShardRecord;
21use super::record::{GateBaseline, GateRecord, read_baseline, read_record};
22
23/// 2026-09-26: A group's records that carry a shard identity, are for this benchmark and its
24/// required subject, and still stand at `sha`, newest first. Whole-draw records are skipped
25/// here and mentioned by [`whole_draw_note`].
26fn standing_shards(
27    root: &Path,
28    baseline: &GateBaseline,
29    group: &'static str,
30    sha: &str,
31    gate: &super::coverage::GateCoverage,
32) -> Vec<(GateRecord, PathBuf)> {
33    records_newest_first(root, group)
34        .into_iter()
35        .filter_map(|path| {
36            let r = read_record(&path).ok()?;
37            (r.shard().is_some()
38                && record_is_for(&r, group, &path)
39                && record_is_required_subject(baseline, &r, group, &path)
40                && record_still_stands(root, sha, &r, gate))
41            .then_some((r, path))
42        })
43        .collect()
44}
45
46/// 2026-09-26: The per-record checks `check_one` also applies to a plain gate's record, asked of
47/// one shard: the run did not fail, the tree was clean, the signature verifies. Empty means
48/// the shard counts. The shard's own verdict and thresholds are not checked; the aggregate is.
49fn shard_problems(root: &Path, label: &str, record: &GateRecord, path: &Path) -> Vec<String> {
50    let mut problems = Vec::new();
51    if record.frame_status_failed() {
52        problems.push(format!(
53            "{label}: the run itself failed: {}",
54            record.verdict_reason
55        ));
56    }
57    if !record.dirty_paths.is_empty() {
58        problems.push(format!(
59            "{label}: measured from a dirty tree — {} uncommitted invalidation-set \
60             file(s) when the run started ({}), so the binary was not {}",
61            record.dirty_paths.len(),
62            record.dirty_paths.join(", "),
63            record.git_sha
64        ));
65    }
66    if let Err(why) = super::signing::verify_record(root, path, &record.git_sha, record.recorded_at)
67    {
68        problems.push(format!("{label}: {why}"));
69    }
70    problems
71}
72
73fn label(group: &str, shard: (usize, usize)) -> String {
74    format!("{group}[{}/{}]", shard.0, shard.1)
75}
76
77/// 2026-09-26: A group's verdict: the newest complete partition of shard records at `sha`,
78/// every shard clean, tallies aggregated over counts and judged against the group's
79/// thresholds.
80pub(super) fn check_group(
81    root: &Path,
82    group: &'static super::group::BenchmarkGroup,
83    sha: &str,
84) -> GateStatus {
85    use crate::benchmarks::bfcl::aggregate;
86
87    let baseline = match read_baseline(root, group.id) {
88        Ok(b) => b,
89        Err(e) => return GateStatus::Missing(format!("baseline unreadable: {e:#}")),
90    };
91    let Some(gate) = super::coverage::find(group.id) else {
92        return GateStatus::Missing(format!("{} has no coverage entry", group.id));
93    };
94
95    let candidates = standing_shards(root, &baseline, group.id, sha, gate);
96    let shards: Vec<ShardRecord> = candidates
97        .iter()
98        .enumerate()
99        .filter_map(|(handle, (r, _))| {
100            let (index, count) = r.shard()?;
101            Some(ShardRecord {
102                index,
103                count,
104                git_sha: r.git_sha.clone(),
105                recorded_at: r.recorded_at,
106                handle,
107            })
108        })
109        .collect();
110    let partition = match super::group::select_partition(group.id, &shards) {
111        Ok(p) => p,
112        Err(fault) => {
113            let mut why = vec![fault.to_string()];
114            if let Some(note) = whole_draw_note(root, group) {
115                why.push(note);
116            }
117            if let Some(note) = why_stale(root, group.id, sha, gate) {
118                why.push(note);
119            }
120            return GateStatus::Missing(why.join(" "));
121        }
122    };
123
124    let count = partition.count;
125    let mut tallies: Vec<BTreeMap<String, aggregate::Tally>> = Vec::new();
126    let mut problems: Vec<String> = Vec::new();
127    let mut newest: Option<GateRecord> = None;
128    for handle in partition.handles {
129        let (record, path) = &candidates[handle];
130        let lbl = label(group.id, record.shard().unwrap_or((0, count)));
131        problems.extend(shard_problems(root, &lbl, record, path));
132        // 2026-09-26: A shard without per-subset tallies cannot be folded in; counting it as
133        // empty would score the group over fewer samples than the draw.
134        let Some(t) = aggregate::tallies_from_metrics(&record.metrics) else {
135            return GateStatus::Missing(format!(
136                "{lbl} has a covering record but no per-subset tallies — it was \
137                 measured by a binary older than the shard split, so the group \
138                 cannot be aggregated. Re-run it at this commit."
139            ));
140        };
141        let errs = record
142            .metrics
143            .get("transport_errors")
144            .copied()
145            .unwrap_or(0.0);
146        if errs > 0.0 {
147            return GateStatus::Fail(vec![format!(
148                "{lbl} recorded {errs:.0} transport failures. Each was scored as \
149                 \"made no call\", which is the CORRECT answer on the irrelevance \
150                 subsets, so a degraded shard can raise the aggregate while \
151                 measuring less of the draw. Re-run it."
152            )]);
153        }
154        tallies.push(t);
155        if newest
156            .as_ref()
157            .is_none_or(|n| record.recorded_at > n.recorded_at)
158        {
159            newest = Some(record.clone());
160        }
161    }
162    if !problems.is_empty() {
163        return GateStatus::Fail(problems);
164    }
165
166    let agg = aggregate::aggregate(&aggregate::union(&tallies));
167    let Some(mut record) = newest else {
168        return GateStatus::Missing(format!("{} has no shards", group.id));
169    };
170    // 2026-09-26: Judge the aggregate in the newest shard's record, which supplies the
171    // checkpoint, serve overrides and hardware; every shard is at the partition's commit.
172    record.benchmark_id = group.id.to_string();
173    record
174        .metrics
175        .insert("overall_accuracy".into(), agg.overall_accuracy);
176    record.metrics.insert(
177        "normalized_single_turn_score".into(),
178        agg.normalized_single_turn_score,
179    );
180    record
181        .metrics
182        .insert("samples".into(), agg.total_samples as f64);
183    record.metrics.insert("shard.count".into(), count as f64);
184
185    match super::scoring::check_record(&record, &baseline) {
186        None => GateStatus::Pass,
187        Some(breaches) => GateStatus::Fail(breaches),
188    }
189}
190
191/// 2026-09-26: The `(index, count)` shards of `group` a certification at `sha` still owes, for a
192/// planner that wants `wanted` shards:
193/// - nothing, when the clean standing shards already form a complete partition;
194/// - else the indices missing from the most complete partition measured at `sha`, whatever
195///   its count (a partition is never assembled across commits);
196/// - else `0..wanted`.
197///
198/// A shard that fails `shard_problems` is owed again. A group with no readable baseline or
199/// coverage entry owes `0..wanted`.
200pub fn shards_owed(
201    root: &Path,
202    group: &'static super::group::BenchmarkGroup,
203    sha: &str,
204    wanted: usize,
205) -> Vec<(usize, usize)> {
206    let fresh = |n: usize| (0..n).map(|i| (i, n)).collect::<Vec<_>>();
207    let (Ok(baseline), Some(gate)) = (
208        read_baseline(root, group.id),
209        super::coverage::find(group.id),
210    ) else {
211        return fresh(wanted);
212    };
213    let candidates = standing_shards(root, &baseline, group.id, sha, gate);
214    let clean: Vec<ShardRecord> = candidates
215        .iter()
216        .enumerate()
217        .filter_map(|(handle, (record, path))| {
218            let (index, count) = record.shard()?;
219            shard_problems(root, &label(group.id, (index, count)), record, path)
220                .is_empty()
221                .then(|| ShardRecord {
222                    index,
223                    count,
224                    git_sha: record.git_sha.clone(),
225                    recorded_at: record.recorded_at,
226                    handle,
227                })
228        })
229        .collect();
230    if super::group::select_partition(group.id, &clean).is_ok() {
231        return Vec::new();
232    }
233    // 2026-09-26: Finish the partition at this commit that is closest to done.
234    let held = super::group::held_by(&clean);
235    match held
236        .iter()
237        .filter(|((_, commit), _)| commit.starts_with(sha) || sha.starts_with(commit.as_str()))
238        .max_by_key(|((count, _), idx)| (idx.len() * 1000 / *count, *count))
239    {
240        Some(((count, _), idx)) => (0..*count)
241            .filter(|i| !idx.contains_key(i))
242            .map(|i| (i, *count))
243            .collect(),
244        None => fresh(wanted),
245    }
246}
247
248/// 2026-09-26: Why the group's newest shard record does not stand: the paths that invalidated
249/// it, or a build-input mismatch. `None` when there is no shard record, it stands, or git
250/// cannot diff.
251fn why_stale(
252    root: &Path,
253    group: &str,
254    sha: &str,
255    gate: &super::coverage::GateCoverage,
256) -> Option<String> {
257    let newest = records_newest_first(root, group)
258        .into_iter()
259        .find_map(|p| {
260            read_record(&p)
261                .ok()
262                .filter(|r| r.shard().is_some())
263                .map(|r| (r, p))
264        })?;
265    let (record, path) = newest;
266    let name = path
267        .file_name()
268        .unwrap_or_default()
269        .to_string_lossy()
270        .into_owned();
271    if record_still_stands(root, sha, &record, gate) {
272        return None;
273    }
274    let why = super::check_paths::invalidating_paths(root, sha, &record.git_sha, gate)?;
275    Some(if why.is_empty() {
276        format!(
277            "Newest shard record is for {} ({name}) — its recorded build inputs do not \
278             match this commit.",
279            record.git_sha
280        )
281    } else {
282        format!(
283            "Newest shard record is for {} ({name}) — invalidated by {}.",
284            record.git_sha,
285            super::check_fmt::summarize_paths(&why)
286        )
287    })
288}
289
290/// 2026-09-26: The note appended when a group directory holds whole-draw records but no
291/// complete partition: those records do not satisfy the gate.
292fn whole_draw_note(root: &Path, group: &super::group::BenchmarkGroup) -> Option<String> {
293    let whole: Vec<PathBuf> = records_newest_first(root, group.id)
294        .into_iter()
295        .filter(|p| read_record(p).is_ok_and(|r| r.shard().is_none()))
296        .collect();
297    let newest = whole.first()?;
298    Some(format!(
299        "The {} whole-draw record(s) under {} (newest {}) no longer satisfy the gate: since \
300         2026-09-13 a group is certified only by a complete partition of shards — see \
301         gate::group.",
302        whole.len(),
303        group.id,
304        newest.file_name().unwrap_or_default().to_string_lossy(),
305    ))
306}
metrale_bench/gate/check_group.rs

metrale_bench/gate/
check_group.rs
