metrale_bench/hardware/
equivalence.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Whether two boxes count as one box for Speed-class numbers: same
4//! GPU name and driver major, clock ceiling and memory within the class's spreads,
5//! no thermal throttle reason asserted, and hottest chassis zones within its delta.
6//!
7//! Owner: bench hardware.
8//! Invariants:
9//! - [`equivalent`] returns `Ok` only when every field but `postcheck_valid` is
10//!   reported on both sides; a field one side lacks is [`Mismatch::Undecidable`].
11//!
12//! `gate::agreement` applies [`equivalent`] to the Speed records of different
13//! signers, and that is the verdict. `met bench certify` calls it only to report
14//! whether its nodes look alike (`remote/schedule.rs` `speed_mode` runs the Speed
15//! class on one node either way).
16
17use super::state::HardwareState;
18use super::{Hardware, HardwareStateReport};
19use crate::gate::GateRecord;
20use serde::{Deserialize, Serialize};
21
22/// 2026-09-26: What decides equivalence, read off a record or a live box.
23#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
24pub struct HardwareFingerprint {
25    /// 2026-09-26: The accelerator name as the driver reports it (`NVIDIA GB10`).
26    pub gpu: String,
27    /// 2026-09-26: The driver's major version (`580` of `580.95.05`).
28    pub driver_major: Option<u32>,
29    /// 2026-09-26: The box's own SM clock ceiling (`clocks.max.sm`), MHz.
30    pub sm_clock_max_mhz: Option<f64>,
31    /// 2026-09-26: `MemTotal` from `/proc/meminfo`, kB.
32    pub mem_total_kb: Option<u64>,
33    /// 2026-09-26: [`super::state::ThrottleActive::thermal`] at capture.
34    pub thermal_alert: Option<bool>,
35    /// 2026-09-26: The hottest chassis zone at capture, °C.
36    pub hottest_chassis_c: Option<f64>,
37    /// 2026-09-26: For a record: whether its postcheck was [`super::policy::Validity::Valid`].
38    /// `None` for a live box, or a record without a report or postcheck.
39    pub postcheck_valid: Option<bool>,
40}
41
42impl HardwareFingerprint {
43    /// 2026-09-26: From a record: static fields off `hardware`, live fields off the
44    /// `before` capture, validity off the postcheck.
45    pub fn from_record(record: &GateRecord) -> Self {
46        let report: Option<&HardwareStateReport> = record.hardware_state.as_ref();
47        let before = report.map(|r| &r.before);
48        let mut fp = Self::from_parts(&record.hardware, before);
49        fp.postcheck_valid = report.and_then(|r| {
50            r.postcheck
51                .as_ref()
52                .map(|p| p.validity == super::policy::Validity::Valid)
53        });
54        fp
55    }
56
57    /// 2026-09-26: From a live box, before anything runs.
58    pub fn from_live(hardware: &Hardware, state: &HardwareState) -> Self {
59        Self::from_parts(hardware, Some(state))
60    }
61
62    fn from_parts(hardware: &Hardware, state: Option<&HardwareState>) -> Self {
63        Self {
64            gpu: hardware.gpu.clone(),
65            driver_major: driver_major(&hardware.driver),
66            sm_clock_max_mhz: state.and_then(|s| s.sm_clock_max_mhz),
67            mem_total_kb: state.and_then(|s| s.mem_total_kb),
68            thermal_alert: state.and_then(|s| s.throttle_active.thermal()),
69            hottest_chassis_c: state.and_then(HardwareState::hottest_chassis_c),
70            postcheck_valid: None,
71        }
72    }
73}
74
75/// 2026-09-26: `580.95.05` → `580`. Empty or non-numeric → `None`.
76pub fn driver_major(driver: &str) -> Option<u32> {
77    driver.split('.').next()?.trim().parse().ok()
78}
79
80/// 2026-09-26: The tolerances, read from the class's `HARDWARE.toml` by [`EquivalencePolicy::speed`].
81#[derive(Clone, Copy, Debug, PartialEq)]
82pub struct EquivalencePolicy {
83    /// 2026-09-26: Largest `|a - b| / max(a, b)` on the clock ceiling.
84    pub clock_spread: f64,
85    /// 2026-09-26: Largest `|a - b| / max(a, b)` on memory.
86    pub mem_spread: f64,
87    /// 2026-09-26: Largest `|a - b|` on the hottest chassis zone, °C.
88    pub chassis_delta_c: f64,
89}
90
91impl EquivalencePolicy {
92    /// 2026-09-26: The policy for one hardware class, from its declared limits
93    /// (`kernels/<hw>/HARDWARE.toml` `[benchmarks.limits.equivalence]` and
94    /// `.thermal`): clock and memory spreads and the chassis delta, on GB10 1 %,
95    /// 5 % and 15 °C.
96    #[must_use]
97    pub fn speed(limits: &super::limits::Limits) -> Self {
98        Self {
99            clock_spread: limits.equivalence.clock_spread,
100            mem_spread: limits.equivalence.mem_spread,
101            chassis_delta_c: limits.thermal.chassis_equivalence_delta_c,
102        }
103    }
104
105    /// 2026-09-26: The policy for `hardware`; `None` when it declares no limits.
106    ///
107    /// # Errors
108    /// Whatever [`super::limits::limits`] refuses: a missing, malformed or
109    /// self-contradicting `HARDWARE.toml`.
110    pub fn speed_for(root: &std::path::Path, hardware: &str) -> anyhow::Result<Option<Self>> {
111        Ok(super::limits::limits(root, hardware)?.map(|l| Self::speed(&l)))
112    }
113}
114
115/// 2026-09-26: Why two fingerprints are not one box.
116#[derive(Clone, Debug, PartialEq)]
117pub enum Mismatch {
118    Gpu(String, String),
119    DriverMajor(u32, u32),
120    ClockSpread {
121        a: f64,
122        b: f64,
123        limit: f64,
124    },
125    MemSpread {
126        a: u64,
127        b: u64,
128        limit: f64,
129    },
130    ChassisDelta {
131        a: f64,
132        b: f64,
133        limit: f64,
134    },
135    /// 2026-09-26: One side (or both) had a thermal reason asserted.
136    ThermalAlert {
137        a: bool,
138        b: bool,
139    },
140    /// 2026-09-26: A record's postcheck was not valid.
141    PostcheckInvalid,
142    /// 2026-09-26: A field one side did not report, named so the operator can fix
143    /// the capture rather than guess.
144    Undecidable(&'static str),
145}
146
147impl std::fmt::Display for Mismatch {
148    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
149        match self {
150            Self::Gpu(a, b) => write!(f, "gpu {a:?} vs {b:?}"),
151            Self::DriverMajor(a, b) => write!(f, "driver major {a} vs {b}"),
152            Self::ClockSpread { a, b, limit } => write!(
153                f,
154                "clock ceiling {a:.0} vs {b:.0} MHz (limit {:.1} %)",
155                limit * 100.0
156            ),
157            Self::MemSpread { a, b, limit } => write!(
158                f,
159                "memory {} vs {} MiB (limit {:.0} %)",
160                a >> 10,
161                b >> 10,
162                limit * 100.0
163            ),
164            Self::ChassisDelta { a, b, limit } => {
165                write!(f, "chassis {a:.0} vs {b:.0} °C (limit {limit:.0} °C)")
166            }
167            Self::ThermalAlert { a, b } => write!(
168                f,
169                "thermal throttle asserted ({})",
170                match (a, b) {
171                    (true, true) => "both",
172                    (true, false) => "first",
173                    _ => "second",
174                }
175            ),
176            Self::PostcheckInvalid => write!(f, "a post-run hardware check was not valid"),
177            Self::Undecidable(field) => write!(f, "{field} not reported on one side"),
178        }
179    }
180}
181
182fn spread(a: f64, b: f64) -> f64 {
183    let m = a.max(b);
184    if m <= 0.0 { 0.0 } else { (a - b).abs() / m }
185}
186
187/// 2026-09-26: Are `a` and `b` one box for Speed-class numbers?
188///
189/// Every mismatch is returned, not the first. A postcheck that is present and
190/// not valid on either side is a mismatch; an absent one (a live box) is not.
191pub fn equivalent(
192    a: &HardwareFingerprint,
193    b: &HardwareFingerprint,
194    p: &EquivalencePolicy,
195) -> Result<(), Vec<Mismatch>> {
196    let mut out = Vec::new();
197    if a.gpu.is_empty() || b.gpu.is_empty() {
198        out.push(Mismatch::Undecidable("gpu"));
199    } else if a.gpu != b.gpu {
200        out.push(Mismatch::Gpu(a.gpu.clone(), b.gpu.clone()));
201    }
202    match (a.driver_major, b.driver_major) {
203        (Some(x), Some(y)) if x != y => out.push(Mismatch::DriverMajor(x, y)),
204        (Some(_), Some(_)) => {}
205        _ => out.push(Mismatch::Undecidable("driver")),
206    }
207    match (a.sm_clock_max_mhz, b.sm_clock_max_mhz) {
208        (Some(x), Some(y)) => {
209            if spread(x, y) > p.clock_spread {
210                out.push(Mismatch::ClockSpread {
211                    a: x,
212                    b: y,
213                    limit: p.clock_spread,
214                });
215            }
216        }
217        _ => out.push(Mismatch::Undecidable("sm_clock_max_mhz")),
218    }
219    match (a.mem_total_kb, b.mem_total_kb) {
220        (Some(x), Some(y)) => {
221            if spread(x as f64, y as f64) > p.mem_spread {
222                out.push(Mismatch::MemSpread {
223                    a: x,
224                    b: y,
225                    limit: p.mem_spread,
226                });
227            }
228        }
229        _ => out.push(Mismatch::Undecidable("mem_total_kb")),
230    }
231    match (a.thermal_alert, b.thermal_alert) {
232        (Some(x), Some(y)) => {
233            if x || y {
234                out.push(Mismatch::ThermalAlert { a: x, b: y });
235            }
236        }
237        _ => out.push(Mismatch::Undecidable("throttle reasons")),
238    }
239    match (a.hottest_chassis_c, b.hottest_chassis_c) {
240        (Some(x), Some(y)) => {
241            if (x - y).abs() > p.chassis_delta_c {
242                out.push(Mismatch::ChassisDelta {
243                    a: x,
244                    b: y,
245                    limit: p.chassis_delta_c,
246                });
247            }
248        }
249        _ => out.push(Mismatch::Undecidable("chassis temperature")),
250    }
251    if a.postcheck_valid == Some(false) || b.postcheck_valid == Some(false) {
252        out.push(Mismatch::PostcheckInvalid);
253    }
254    if out.is_empty() { Ok(()) } else { Err(out) }
255}
256
257#[cfg(test)]
258#[path = "equivalence_tests.rs"]
259mod equivalence_tests;
metrale_bench/hardware/equivalence.rs

metrale_bench/hardware/
equivalence.rs
