metrale_bench/hardware/
state.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: [`HardwareState`], what the box was doing at one instant, and
4//! [`HardwareStateDelta`], what changed between the captures before and after a run.
5//!
6//! Owner: bench hardware.
7//! Invariants:
8//! - A reading that does not parse is `None`, never a default number (the parsers
9//!   in [`super::parse`]).
10//! - A throttle counter that went backwards between captures yields `None` in the
11//!   delta, never zero.
12//!
13//! [`super::Hardware`] answers "which box"; this answers "what state was that box in".
14//! Two boxes with identical fingerprints can differ in chassis temperature, throttle
15//! time and clock under load, and only the state shows it.
16//!
17//! Memory comes from `/proc/meminfo`: nvidia-smi reports GB10's framebuffer usage as
18//! `N/A` (`fixtures/gb10_performance.txt`). `Cached` is kept beside `MemAvailable`
19//! rather than folded into it.
20
21use serde::{Deserialize, Serialize};
22
23/// 2026-09-26: One `/sys/class/thermal/thermal_zone*` reading.
24#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
25pub struct ThermalZone {
26    /// 2026-09-26: The zone's `type` file, trimmed. Empty when unreadable.
27    pub name: String,
28    pub temp_c: f64,
29}
30
31/// 2026-09-26: Cumulative throttle time per reason, in the microseconds
32/// `nvidia-smi -q -d PERFORMANCE` reports under `Clocks Event Reasons Counters`.
33///
34/// These are counters, not flags: the useful reading is the difference across a
35/// run (see [`HardwareStateDelta`]), because a box can start cool and throttle in
36/// the middle. `None` means the reason was not reported; `Some(0)` means it was
37/// reported as never having fired.
38#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
39pub struct ThrottleCounters {
40    #[serde(default, skip_serializing_if = "Option::is_none")]
41    pub sw_power_cap_us: Option<u64>,
42    #[serde(default, skip_serializing_if = "Option::is_none")]
43    pub sw_thermal_us: Option<u64>,
44    #[serde(default, skip_serializing_if = "Option::is_none")]
45    pub hw_thermal_us: Option<u64>,
46    #[serde(default, skip_serializing_if = "Option::is_none")]
47    pub hw_power_brake_us: Option<u64>,
48    #[serde(default, skip_serializing_if = "Option::is_none")]
49    pub sync_boost_us: Option<u64>,
50}
51
52/// 2026-09-26: Throttle reasons asserted at the instant of capture.
53///
54/// The pre-run check ([`super::policy::precheck`]) reads these; the post-run
55/// check reads the counters' delta.
56#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
57pub struct ThrottleActive {
58    #[serde(default, skip_serializing_if = "Option::is_none")]
59    pub sw_power_cap: Option<bool>,
60    #[serde(default, skip_serializing_if = "Option::is_none")]
61    pub sw_thermal: Option<bool>,
62    #[serde(default, skip_serializing_if = "Option::is_none")]
63    pub hw_thermal: Option<bool>,
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub hw_power_brake: Option<bool>,
66}
67
68impl ThrottleActive {
69    /// 2026-09-26: True when SW thermal, HW thermal or the HW power brake is
70    /// asserted. `None` when none of the three was reported, so the caller can
71    /// tell "not throttling" from "no idea".
72    ///
73    /// SW power cap is excluded: a power-limited part accrues it in normal
74    /// running (16,130 s on the cool box's counter in
75    /// `fixtures/gb10_performance.txt`), so it says nothing about a thermal fault.
76    pub fn thermal(&self) -> Option<bool> {
77        match (self.sw_thermal, self.hw_thermal, self.hw_power_brake) {
78            (None, None, None) => None,
79            (sw, hw, brake) => {
80                Some(sw.unwrap_or(false) || hw.unwrap_or(false) || brake.unwrap_or(false))
81            }
82        }
83    }
84}
85
86/// 2026-09-26: One GPU-resident compute process, as `--query-compute-apps` reports it.
87#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
88pub struct GpuComputeApp {
89    pub pid: u32,
90    pub name: String,
91    /// 2026-09-26: `None` when the memory cell does not parse, such as `[N/A]`.
92    #[serde(default, skip_serializing_if = "Option::is_none")]
93    pub used_mib: Option<u64>,
94}
95
96/// 2026-09-26: Which individual box this is. [`super::Hardware::gate_key`] gives
97/// the silicon class (`"gb10"`) that baselines are keyed by; this tells two boxes
98/// of one class apart.
99#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
100pub struct MachineIdentity {
101    #[serde(default, skip_serializing_if = "Option::is_none")]
102    pub hostname: Option<String>,
103    /// 2026-09-26: `/etc/machine-id`, which a hostname change does not alter.
104    #[serde(default, skip_serializing_if = "Option::is_none")]
105    pub machine_id: Option<String>,
106    #[serde(default, skip_serializing_if = "Option::is_none")]
107    pub gpu: Option<String>,
108    #[serde(default, skip_serializing_if = "Option::is_none")]
109    pub driver: Option<String>,
110}
111
112impl MachineIdentity {
113    /// 2026-09-26: `"gb10@dgx1"`: silicon class, then hostname.
114    ///
115    /// An absent half reads `"unknown"` rather than being dropped, so two records
116    /// with different missing halves never collapse to the same string.
117    pub fn perf_class(&self) -> String {
118        let class = self
119            .gpu
120            .as_deref()
121            .map(|g| super::Hardware {
122                gpu: g.to_string(),
123                ..super::Hardware::default()
124            })
125            .map(|h| h.gate_key())
126            .unwrap_or_else(|| "unknown".to_string());
127        let host = self.hostname.as_deref().unwrap_or("unknown");
128        format!("{class}@{host}")
129    }
130}
131
132/// 2026-09-26: Everything the collector could read about the box at one instant.
133/// The executor captures it before a run and again when the run's terminal frame
134/// is ready; [`HardwareStateDelta`] is the difference.
135#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
136pub struct HardwareState {
137    /// 2026-09-26: Unix seconds; `0` when the system clock reads before the epoch.
138    #[serde(default)]
139    pub captured_at: u64,
140    #[serde(default)]
141    pub machine: MachineIdentity,
142    #[serde(default, skip_serializing_if = "Option::is_none")]
143    pub gpu_temp_c: Option<f64>,
144    /// 2026-09-26: Every `/sys/class/thermal` zone whose temperature parsed, in
145    /// zone-number order. `None` means the directory could not be read, not
146    /// "this box has no zones".
147    #[serde(default, skip_serializing_if = "Option::is_none")]
148    pub chassis_temps_c: Option<Vec<ThermalZone>>,
149    #[serde(default, skip_serializing_if = "Option::is_none")]
150    pub sm_clock_mhz: Option<f64>,
151    #[serde(default, skip_serializing_if = "Option::is_none")]
152    pub sm_clock_max_mhz: Option<f64>,
153    #[serde(default)]
154    pub throttle_counters: ThrottleCounters,
155    #[serde(default)]
156    pub throttle_active: ThrottleActive,
157    /// 2026-09-26: `MemAvailable` from `/proc/meminfo`, kB.
158    #[serde(default, skip_serializing_if = "Option::is_none")]
159    pub mem_available_kb: Option<u64>,
160    #[serde(default, skip_serializing_if = "Option::is_none")]
161    pub mem_total_kb: Option<u64>,
162    /// 2026-09-26: `Cached` from `/proc/meminfo`, kB.
163    #[serde(default, skip_serializing_if = "Option::is_none")]
164    pub page_cache_kb: Option<u64>,
165    /// 2026-09-26: The `--query-compute-apps` rows. When that query prints nothing
166    /// or fails, `Some(vec![])` if `--list-gpus` answers, else `None`. The
167    /// precheck warns on `None` and accepts an empty list.
168    #[serde(default, skip_serializing_if = "Option::is_none")]
169    pub gpu_compute_apps: Option<Vec<GpuComputeApp>>,
170    #[serde(default, skip_serializing_if = "Option::is_none")]
171    pub cpu_governor: Option<String>,
172    #[serde(default, skip_serializing_if = "Option::is_none")]
173    pub persistence_mode: Option<bool>,
174    /// 2026-09-26: Which collectors answered, from `"nvidia-smi"`, `"procfs"` and
175    /// `"sysfs"`. Empty means none did.
176    #[serde(default, skip_serializing_if = "Vec::is_empty")]
177    pub sources: Vec<String>,
178}
179
180impl HardwareState {
181    /// 2026-09-26: Read the local box; see [`super::collect`]. A source that fails
182    /// leaves its fields unset rather than failing the capture.
183    pub fn collect() -> Self {
184        super::collect::collect()
185    }
186
187    /// 2026-09-26: GPU compute processes other than this process, so a server
188    /// running in this process is not counted and a separate server process is.
189    /// `None` when the list could not be read. The precheck refuses a Speed run
190    /// above [`super::policy::MAX_FOREIGN_COMPUTE_APPS`].
191    pub fn foreign_compute_apps(&self) -> Option<usize> {
192        let own = std::process::id();
193        Some(
194            self.gpu_compute_apps
195                .as_ref()?
196                .iter()
197                .filter(|a| a.pid != own)
198                .count(),
199        )
200    }
201
202    /// 2026-09-26: The hottest chassis zone; `None` when no zone was read.
203    pub fn hottest_chassis_c(&self) -> Option<f64> {
204        self.chassis_temps_c
205            .as_ref()?
206            .iter()
207            .map(|z| z.temp_c)
208            .fold(None::<f64>, |acc, t| Some(acc.map_or(t, |a| a.max(t))))
209    }
210
211    /// 2026-09-26: Current SM clock as a fraction of the box's own maximum;
212    /// `None` when either is missing or the maximum is not positive.
213    ///
214    /// Meaningful only under load: the idle GB10 fixture reads 208 of 3003 MHz
215    /// (`fixtures/gb10_query_gpu.csv`).
216    pub fn clock_headroom(&self) -> Option<f64> {
217        let max = self.sm_clock_max_mhz?;
218        (max > 0.0).then(|| self.sm_clock_mhz.map(|c| c / max))?
219    }
220
221    /// 2026-09-26: One line for a run log.
222    pub fn one_line(&self) -> String {
223        let mut parts = vec![self.machine.perf_class()];
224        if let Some(t) = self.gpu_temp_c {
225            parts.push(format!("gpu {t:.0} °C"));
226        }
227        if let Some(t) = self.hottest_chassis_c() {
228            parts.push(format!("chassis max {t:.0} °C"));
229        }
230        match (self.sm_clock_mhz, self.sm_clock_max_mhz) {
231            (Some(c), Some(m)) => parts.push(format!("sm {c:.0}/{m:.0} MHz")),
232            (Some(c), None) => parts.push(format!("sm {c:.0} MHz")),
233            _ => {}
234        }
235        match self.foreign_compute_apps() {
236            Some(n) => parts.push(format!("{n} foreign gpu proc")),
237            None => parts.push("foreign gpu proc unknown".to_string()),
238        }
239        if let Some(kb) = self.mem_available_kb {
240            parts.push(format!("{:.1} GiB avail", kb as f64 / 1_048_576.0));
241        }
242        parts.join(" · ")
243    }
244}
245
246/// 2026-09-26: What changed across the run.
247///
248/// A box that starts cool and throttles midway has a healthy pre-state; whether a
249/// throttle counter advanced while the benchmark ran is visible only in the
250/// difference of two captures.
251#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
252pub struct HardwareStateDelta {
253    #[serde(default, skip_serializing_if = "Option::is_none")]
254    pub elapsed_s: Option<u64>,
255    #[serde(default, skip_serializing_if = "Option::is_none")]
256    pub sw_power_cap_us: Option<u64>,
257    #[serde(default, skip_serializing_if = "Option::is_none")]
258    pub sw_thermal_us: Option<u64>,
259    #[serde(default, skip_serializing_if = "Option::is_none")]
260    pub hw_thermal_us: Option<u64>,
261    #[serde(default, skip_serializing_if = "Option::is_none")]
262    pub hw_power_brake_us: Option<u64>,
263    #[serde(default, skip_serializing_if = "Option::is_none")]
264    pub gpu_temp_delta_c: Option<f64>,
265    #[serde(default, skip_serializing_if = "Option::is_none")]
266    pub hottest_chassis_delta_c: Option<f64>,
267}
268
269/// 2026-09-26: `after - before`; `None` when either is missing or the counter went
270/// backwards (a driver reload mid-run), which is an unusable reading, not zero.
271fn advance(before: Option<u64>, after: Option<u64>) -> Option<u64> {
272    let (b, a) = (before?, after?);
273    // 2026-09-26: `checked_sub`, not `(a >= b).then_some(a - b)`: the argument to
274    // `then_some` is evaluated eagerly, so the backwards case would panic in a
275    // debug build before the guard runs.
276    a.checked_sub(b)
277}
278
279impl HardwareStateDelta {
280    pub fn between(before: &HardwareState, after: &HardwareState) -> Self {
281        let (b, a) = (&before.throttle_counters, &after.throttle_counters);
282        Self {
283            elapsed_s: after.captured_at.checked_sub(before.captured_at),
284            sw_power_cap_us: advance(b.sw_power_cap_us, a.sw_power_cap_us),
285            sw_thermal_us: advance(b.sw_thermal_us, a.sw_thermal_us),
286            hw_thermal_us: advance(b.hw_thermal_us, a.hw_thermal_us),
287            hw_power_brake_us: advance(b.hw_power_brake_us, a.hw_power_brake_us),
288            gpu_temp_delta_c: Option::zip(before.gpu_temp_c, after.gpu_temp_c).map(|(b, a)| a - b),
289            hottest_chassis_delta_c: Option::zip(
290                before.hottest_chassis_c(),
291                after.hottest_chassis_c(),
292            )
293            .map(|(b, a)| a - b),
294        }
295    }
296
297    /// 2026-09-26: Did a thermal throttle counter (SW thermal, HW thermal, HW power
298    /// brake) accumulate time during the run?
299    ///
300    /// `Some(true)` when any readable one advanced; `Some(false)` only when all
301    /// three were readable and none advanced; otherwise `None`, which the policy
302    /// renders as unknown. SW power cap is excluded for the reason given on
303    /// [`ThrottleActive::thermal`].
304    pub fn thermal_throttle_advanced(&self) -> Option<bool> {
305        let counters = [
306            self.sw_thermal_us,
307            self.hw_thermal_us,
308            self.hw_power_brake_us,
309        ];
310        if counters.into_iter().flatten().any(|value| value > 0) {
311            return Some(true);
312        }
313        counters
314            .into_iter()
315            .all(|value| value == Some(0))
316            .then_some(false)
317    }
318
319    /// 2026-09-26: True when every counter [`Self::thermal_throttle_fraction`] sums
320    /// was readable on both captures.
321    ///
322    /// When false the fraction is a lower bound, and the caller must say so: an
323    /// unreadable counter contributes nothing to the sum.
324    pub fn thermal_counters_complete(&self) -> bool {
325        self.sw_thermal_us.is_some()
326            && self.hw_thermal_us.is_some()
327            && self.hw_power_brake_us.is_some()
328    }
329
330    /// 2026-09-26: The throttled fraction of the run: the readable thermal
331    /// counters' sum over the elapsed time.
332    ///
333    /// `None` when the run has no positive duration, or when no thermal counter
334    /// was readable on both captures. A lower bound when only some were; see
335    /// [`Self::thermal_counters_complete`].
336    pub fn thermal_throttle_fraction(&self) -> Option<f64> {
337        let secs = self.elapsed_s.filter(|s| *s > 0)? as f64;
338        let counters = [
339            self.sw_thermal_us,
340            self.hw_thermal_us,
341            self.hw_power_brake_us,
342        ];
343        if counters.iter().all(Option::is_none) {
344            return None;
345        }
346        let us: u64 = counters.into_iter().flatten().sum();
347        Some(us as f64 / 1e6 / secs)
348    }
349}
350
351#[cfg(test)]
352#[path = "state_tests.rs"]
353mod tests;
metrale_bench/hardware/state.rs

metrale_bench/hardware/
state.rs
