metrale_bench/gate/agreement.rs
1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Do the records a PR adds agree with each other and with the
4//! head being certified?
5//!
6//! Each record is bound to its commit by its signature. `check` adds two
7//! rules over the whole added set:
8//!
9//! - Every record must stand at the head (`Standing::Stands`, from
10//! `check::record_standing`): the diff from the record's commit to the head
11//! touches nothing its gate measures, or the record's closure excuses what
12//! it touches. The rule is content, not ancestry, so records measured at
13//! different commits may be added together.
14//! - Speed-class records (`Sensitivity::Speed`, read from the registry) from
15//! more than one signer are accepted only when every cross-signer pair is one
16//! box by `hardware::equivalence::equivalent`, judged from the
17//! records' own `hardware` and `hardware_state` captures under the class's
18//! `kernels/<hw>/HARDWARE.toml` limits. Correctness-class records may come
19//! from any number of signers.
20//!
21//! The one-box rule for Speed rests on a measurement taken 2026-09-06 on one
22//! gate at one commit: dgx2 22.78 tok/s (sigma 0.063, n=10) against dgx3 23.44
23//! (sigma 0.070, n=10), a gap ten times either box's sigma.
24//!
25//! Whether a signer is committed in `.github/record-signers/` is checked by
26//! `gate::signing`, not here.
27//!
28//! Owner: bench gate.
29//! Invariants:
30//! - A benchmark id the registry does not know is reported as
31//! `Disagreement::UnknownBenchmark`, never given a class.
32//! - `check` returns every disagreement it finds, not only the first.
33
34use super::check::Standing;
35use super::coverage;
36use crate::hardware::equivalence::{EquivalencePolicy, HardwareFingerprint, equivalent};
37use crate::hardware::policy::Sensitivity;
38use crate::registry;
39
40/// 2026-09-26: One record as the agreement rule sees it.
41#[derive(Debug, Clone, PartialEq)]
42pub struct AddedRecord {
43 /// 2026-09-26: Path, for messages only; never parsed.
44 pub path: String,
45 pub benchmark_id: String,
46 pub git_sha: String,
47 /// 2026-09-26: The signing key fingerprint from the `.sig` sidecar.
48 pub signer: String,
49 /// 2026-09-26: What the record says about the box it was measured on.
50 /// `None` makes the record equivalent to nothing in [`check`].
51 pub hardware: Option<HardwareFingerprint>,
52 /// 2026-09-26: The box class the record names (`Hardware::gate_key`). In
53 /// a cross-signer pair, the first record's class selects the equivalence
54 /// policy.
55 pub hardware_class: String,
56 /// 2026-09-26: Where the record stands at the head being certified,
57 /// computed by the caller, e.g. with [`standing_at`].
58 pub standing: Standing,
59}
60
61/// 2026-09-26: Why a set of added records does not hang together.
62#[derive(Debug, Clone, PartialEq, Eq)]
63pub enum Disagreement {
64 /// 2026-09-26: A record whose standing is not `Stands`: its commit cannot
65 /// be diffed against the head, or the diff touches what its gate
66 /// measures. Reported for every class.
67 Straggler {
68 path: String,
69 git_sha: String,
70 why: String,
71 },
72 /// 2026-09-26: Speed-class records signed by more than one identity, on
73 /// boxes the records themselves do not show to be equivalent.
74 SpeedSigners {
75 /// 2026-09-26: The gates in a non-equivalent pair, sorted and
76 /// deduplicated.
77 gates: Vec<String>,
78 signers: Vec<String>,
79 /// 2026-09-26: Why the boxes are not one box, one entry per offending
80 /// pair.
81 mismatches: Vec<String>,
82 },
83 /// 2026-09-26: A record naming a benchmark the registry does not have.
84 UnknownBenchmark(String),
85}
86
87impl std::fmt::Display for Disagreement {
88 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
89 match self {
90 Self::Straggler { path, git_sha, why } => write!(
91 f,
92 "{path} was measured at {git_sha} and does not stand at the head: {why}. \
93 Re-measure it at the head you intend to merge."
94 ),
95 Self::SpeedSigners {
96 gates,
97 signers,
98 mismatches,
99 } => write!(
100 f,
101 "{} speed-class gate(s) ({}) carry {} different signing keys ({}) \
102 and the records do not show the boxes to be equivalent: {}. \
103 Throughput and latency are box-dependent — measured 0.66 tok/s \
104 between two boxes against a within-box sigma of 0.07 — so these \
105 must come from ONE box, or from boxes whose captures agree on \
106 GPU, driver line, clock ceiling, memory and thermal state. \
107 Correctness-class gates may span boxes freely.",
108 gates.len(),
109 gates.join(", "),
110 signers.len(),
111 signers.join(", "),
112 mismatches.join("; ")
113 ),
114 Self::UnknownBenchmark(id) => write!(
115 f,
116 "record names benchmark {id:?}, which is not in the registry — \
117 refusing to classify it. An unrecognised gate must not inherit \
118 the permissive rule."
119 ),
120 }
121 }
122}
123
124/// 2026-09-26: A record's [`Standing`] at `head`, judged against the coverage
125/// entry of its `benchmark_id`. An id with no coverage entry is
126/// [`Standing::Unknown`].
127pub fn standing_at(root: &std::path::Path, head: &str, record: &super::GateRecord) -> Standing {
128 match coverage::find(&record.benchmark_id) {
129 Some(gate) => super::check::record_standing(root, head, record, gate),
130 None => Standing::Unknown,
131 }
132}
133
134fn policy_for(root: &std::path::Path, class: &str) -> anyhow::Result<Option<EquivalencePolicy>> {
135 EquivalencePolicy::speed_for(root, class)
136}
137
138/// 2026-09-26: The class a gate's records belong to, read from the registry
139/// and never from the record, so a record cannot choose its own class.
140pub fn sensitivity_of(benchmark_id: &str) -> Option<Sensitivity> {
141 registry::find(benchmark_id).map(|d| d.sensitivity)
142}
143
144/// 2026-09-26: Check that the records a PR adds agree with one another.
145///
146/// Returns every disagreement found. `root` is where the hardware class's
147/// equivalence policy is read from (`kernels/<hw>/HARDWARE.toml`); a class
148/// with no `[benchmarks.limits]` makes every cross-signer Speed pair a
149/// mismatch.
150pub fn check(root: &std::path::Path, added: &[AddedRecord]) -> Vec<Disagreement> {
151 let mut out = Vec::new();
152 if added.is_empty() {
153 return out;
154 }
155
156 for r in added {
157 let why = match &r.standing {
158 Standing::Stands => continue,
159 Standing::Unknown => {
160 "its commit cannot be diffed against the head (unknown to this repository, \
161 or git failed)"
162 .to_string()
163 }
164 Standing::Invalidated(paths) => format!(
165 "commits since it touched what its gate measures ({})",
166 paths.join(", ")
167 ),
168 };
169 out.push(Disagreement::Straggler {
170 path: r.path.clone(),
171 git_sha: r.git_sha.clone(),
172 why,
173 });
174 }
175
176 let speed: Vec<&AddedRecord> = added
177 .iter()
178 .filter(|r| match sensitivity_of(&r.benchmark_id) {
179 None => {
180 out.push(Disagreement::UnknownBenchmark(r.benchmark_id.clone()));
181 false
182 }
183 Some(Sensitivity::Speed) => true,
184 Some(Sensitivity::Correctness) => false,
185 })
186 .collect();
187 let mut speed_signers: Vec<String> = speed.iter().map(|r| r.signer.clone()).collect();
188 speed_signers.sort();
189 speed_signers.dedup();
190 if speed_signers.len() > 1 {
191 // 2026-09-26: Every cross-signer pair must be one box by the records'
192 // own captures.
193 let mut mismatches = Vec::new();
194 let mut gates = Vec::new();
195 for (i, a) in speed.iter().enumerate() {
196 for b in &speed[i + 1..] {
197 if a.signer == b.signer {
198 continue;
199 }
200 let why = match (&a.hardware, &b.hardware) {
201 (Some(x), Some(y)) => match policy_for(root, &a.hardware_class) {
202 Ok(Some(policy)) => match equivalent(x, y, &policy) {
203 Ok(()) => continue,
204 Err(m) => m
205 .iter()
206 .map(ToString::to_string)
207 .collect::<Vec<_>>()
208 .join(", "),
209 },
210 Ok(None) => format!(
211 "kernels/{}/HARDWARE.toml declares no [benchmarks.limits.thermal] \
212 envelope, so two boxes of that class are never one box",
213 a.hardware_class
214 ),
215 Err(e) => format!("{e:#}"),
216 },
217 _ => "a record carries no hardware capture".to_owned(),
218 };
219 mismatches.push(format!(
220 "{} ({}) vs {} ({}): {why}",
221 a.benchmark_id,
222 short(&a.signer),
223 b.benchmark_id,
224 short(&b.signer)
225 ));
226 gates.push(a.benchmark_id.clone());
227 gates.push(b.benchmark_id.clone());
228 }
229 }
230 if !mismatches.is_empty() {
231 gates.sort();
232 gates.dedup();
233 out.push(Disagreement::SpeedSigners {
234 gates,
235 signers: speed_signers,
236 mismatches,
237 });
238 }
239 }
240 out
241}
242
243fn short(signer: &str) -> &str {
244 &signer[..signer.len().min(12)]
245}
246
247/// 2026-09-26: Every required gate, split into (Speed, Correctness) by the
248/// class that decides its signer rule. A gate the registry does not know is in
249/// neither list.
250pub fn required_by_class() -> (Vec<&'static str>, Vec<&'static str>) {
251 let mut speed = Vec::new();
252 let mut correctness = Vec::new();
253 for g in coverage::REQUIRED.iter() {
254 match sensitivity_of(g.id) {
255 Some(Sensitivity::Speed) => speed.push(g.id),
256 Some(Sensitivity::Correctness) => correctness.push(g.id),
257 None => {}
258 }
259 }
260 (speed, correctness)
261}