metrale_telemetry/
device.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: L0: what the GPU reports through NVML: power, the cumulative
4//! energy counter, clocks, temperature, clock-event reasons, memory, PCIe and
5//! NVLink.
6//!
7//! [`DeviceSource`] is the I/O seam: [`NvmlSource`] reads the driver, and a
8//! test supplies its own source or, for the NVML path itself, its own NVML
9//! function table (`nvml_source_tests`). Energy is the driver's counter
10//! (`nvmlDeviceGetTotalEnergyConsumption`), not an integral of sampled power.
11//!
12//! Owner: telemetry.
13//! Invariants: a field [`NvmlSource`] cannot read is `None`, never 0.
14
15use metrale_gpu_sys::nvml::{
16    NVML_CLOCK_GRAPHICS, NVML_CLOCK_MEM, NVML_CLOCK_SM, NVML_PCIE_UTIL_RX_BYTES,
17    NVML_PCIE_UTIL_TX_BYTES, Nvml, NvmlFns, NvmlUnavailable,
18};
19
20/// 2026-09-26: One reading. `None`: the device or driver does not report the
21/// field, or reading it failed.
22#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)]
23pub struct DeviceReading {
24    pub power_mw: Option<u64>,
25    /// 2026-09-26: The raw cumulative counter, millijoules since driver load.
26    pub energy_counter_mj: Option<u64>,
27    pub graphics_clock_mhz: Option<u64>,
28    pub sm_clock_mhz: Option<u64>,
29    pub mem_clock_mhz: Option<u64>,
30    pub temperature_c: Option<u64>,
31    /// 2026-09-26: NVML clocks-event-reasons bitmask.
32    pub clocks_event_reasons: Option<u64>,
33    pub mem_used_bytes: Option<u64>,
34    pub mem_total_bytes: Option<u64>,
35    pub pcie_tx_kbps: Option<u64>,
36    pub pcie_rx_kbps: Option<u64>,
37    pub nvlink_active_links: Option<u64>,
38}
39
40/// 2026-09-26: The number of [`DeviceReading`] fields; `fields` lists them in
41/// declaration order.
42const FIELDS: usize = 12;
43/// 2026-09-26: Words in the encoded form: a presence mask, the sample time,
44/// the fields.
45pub const SAMPLE_WORDS: usize = FIELDS + 2;
46
47impl DeviceReading {
48    fn fields(&self) -> [Option<u64>; FIELDS] {
49        [
50            self.power_mw,
51            self.energy_counter_mj,
52            self.graphics_clock_mhz,
53            self.sm_clock_mhz,
54            self.mem_clock_mhz,
55            self.temperature_c,
56            self.clocks_event_reasons,
57            self.mem_used_bytes,
58            self.mem_total_bytes,
59            self.pcie_tx_kbps,
60            self.pcie_rx_kbps,
61            self.nvlink_active_links,
62        ]
63    }
64
65    fn from_fields(f: [Option<u64>; FIELDS]) -> Self {
66        Self {
67            power_mw: f[0],
68            energy_counter_mj: f[1],
69            graphics_clock_mhz: f[2],
70            sm_clock_mhz: f[3],
71            mem_clock_mhz: f[4],
72            temperature_c: f[5],
73            clocks_event_reasons: f[6],
74            mem_used_bytes: f[7],
75            mem_total_bytes: f[8],
76            pcie_tx_kbps: f[9],
77            pcie_rx_kbps: f[10],
78            nvlink_active_links: f[11],
79        }
80    }
81
82    /// 2026-09-26: The word form a [`crate::seqcell::SeqCell`] carries.
83    pub fn encode(&self, at_ns: u64) -> [u64; SAMPLE_WORDS] {
84        let mut w = [0u64; SAMPLE_WORDS];
85        w[1] = at_ns;
86        for (i, f) in self.fields().into_iter().enumerate() {
87            if let Some(v) = f {
88                w[0] |= 1 << i;
89                w[i + 2] = v;
90            }
91        }
92        w
93    }
94
95    /// 2026-09-26: Inverse of [`DeviceReading::encode`]: the reading and its time.
96    pub fn decode(w: &[u64; SAMPLE_WORDS]) -> (Self, u64) {
97        let mut f = [None; FIELDS];
98        for (i, slot) in f.iter_mut().enumerate() {
99            if w[0] & (1 << i) != 0 {
100                *slot = Some(w[i + 2]);
101            }
102        }
103        (Self::from_fields(f), w[1])
104    }
105}
106
107/// 2026-09-26: Named bits of the clocks-event-reasons mask, as `nvml.h`
108/// defines them.
109pub const CLOCK_EVENT_REASONS: [(&str, u64); 6] = [
110    ("gpu_idle", 0x1),
111    ("sw_power_cap", 0x4),
112    ("hw_slowdown", 0x8),
113    ("sw_thermal_slowdown", 0x20),
114    ("hw_thermal_slowdown", 0x40),
115    ("hw_power_brake_slowdown", 0x80),
116];
117
118/// 2026-09-26: Where device readings come from.
119pub trait DeviceSource: Send {
120    /// 2026-09-26: One reading, now. Never fails as a whole: an unreadable
121    /// field is `None`.
122    fn read(&mut self) -> DeviceReading;
123    /// 2026-09-26: Failed reads so far. A "not supported" or "no permission"
124    /// answer is not a failure. `NvmlSource` does not count the NVLink state
125    /// queries: a link that does not answer is skipped.
126    fn read_errors(&self) -> u64;
127}
128
129/// 2026-09-26: NVML-backed source for one device ordinal.
130pub struct NvmlSource {
131    nvml: Nvml,
132    index: u32,
133    errors: u64,
134}
135
136impl NvmlSource {
137    /// 2026-09-26: Over the system library. `Err` when NVML is not loadable or
138    /// has no such device.
139    pub fn open(index: u32) -> Result<Self, NvmlUnavailable> {
140        Self::checked(Nvml::open()?, index)
141    }
142
143    /// 2026-09-26: Over an explicit NVML function table.
144    pub fn with_fns(fns: NvmlFns, index: u32) -> Result<Self, NvmlUnavailable> {
145        Self::checked(Nvml::with_fns(fns)?, index)
146    }
147
148    fn checked(nvml: Nvml, index: u32) -> Result<Self, NvmlUnavailable> {
149        nvml.device(index)?;
150        Ok(Self {
151            nvml,
152            index,
153            errors: 0,
154        })
155    }
156}
157
158impl DeviceSource for NvmlSource {
159    fn read(&mut self) -> DeviceReading {
160        let Ok(dev) = self.nvml.device(self.index) else {
161            self.errors += 1;
162            return DeviceReading::default();
163        };
164        let mut errors = 0u64;
165        let mut ok = |r: Result<Option<u64>, i32>| match r {
166            Ok(v) => v,
167            Err(_) => {
168                errors += 1;
169                None
170            }
171        };
172        let widen = |r: Result<Option<u32>, i32>| r.map(|v| v.map(u64::from));
173        let memory = dev.memory();
174        let reading = DeviceReading {
175            power_mw: ok(widen(dev.power_mw())),
176            energy_counter_mj: ok(dev.total_energy_mj()),
177            graphics_clock_mhz: ok(widen(dev.clock_mhz(NVML_CLOCK_GRAPHICS))),
178            sm_clock_mhz: ok(widen(dev.clock_mhz(NVML_CLOCK_SM))),
179            mem_clock_mhz: ok(widen(dev.clock_mhz(NVML_CLOCK_MEM))),
180            temperature_c: ok(widen(dev.temperature_c())),
181            clocks_event_reasons: ok(dev.clocks_event_reasons()),
182            mem_used_bytes: ok(memory.map(|m| m.map(|m| m.used))),
183            mem_total_bytes: memory.ok().flatten().map(|m| m.total),
184            pcie_tx_kbps: ok(widen(dev.pcie_kbps(NVML_PCIE_UTIL_TX_BYTES))),
185            pcie_rx_kbps: ok(widen(dev.pcie_kbps(NVML_PCIE_UTIL_RX_BYTES))),
186            nvlink_active_links: dev.nvlink_active_links().map(u64::from),
187        };
188        self.errors += errors;
189        reading
190    }
191
192    fn read_errors(&self) -> u64 {
193        self.errors
194    }
195}
196
197#[cfg(test)]
198#[path = "nvml_source_tests.rs"]
199mod tests;
metrale_telemetry/device.rs

metrale_telemetry/
device.rs
