metrale_telemetry/lib.rs
1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Engine telemetry: run metrics, kernel audit, launch trace,
4//! startup progress events, and the metal-up instrument set.
5//!
6//! # Levels
7//! [`Level::Off`] (where a new [`Telemetry`] starts), [`Level::Basic`] and
8//! [`Level::Kernel`].
9//!
10//! # Layers
11//! | Layer | Module | Content |
12//! |---|---|---|
13//! | L0 device | [`device`], [`sampler`] | NVML via dlopen, sampled at 10 Hz when serving: power, energy counter, clocks, temperature, clock-event reasons, memory, PCIe, NVLink |
14//! | L1 kernel | [`kernel`] | CUDA-event spans per serve lane each step; per kernel one step in N |
15//! | L2 scheduler | [`sched`] | queue depths, batch shape, loop phases, blocking syncs/D2H, graph replays/captures |
16//! | L3 cache | [`sched`], [`prefix_cache`] | KV blocks, SSM slots, prefix-cache hits/misses |
17//! | L4 spec | [`sched::SpecMatrix`] | verify acceptance by draft depth |
18//! | L5 request | [`request`] | TTFT, TPOT, end-to-end |
19//! | Energy | [`energy`] | live J/token and J/request from NVML counter deltas |
20//!
21//! Exported by [`export`]: Prometheus text, JSONL events, and OTLP behind a
22//! feature.
23//!
24//! Owner: telemetry.
25//! Invariants: at [`Level::Off`], every hot-path entry point of [`Telemetry`]
26//! returns after one relaxed atomic load, with no clock read and no allocation
27//! (`tests/zero_cost_off.rs`).
28
29pub mod clock;
30pub mod device;
31pub mod energy;
32pub mod export;
33pub mod hub;
34pub mod instrument;
35pub mod kernel;
36pub mod kernel_audit;
37pub mod launch_trace;
38pub mod level;
39pub mod prefix_cache;
40pub mod progress;
41pub mod request;
42pub mod run_metrics;
43pub mod sampler;
44pub mod sched;
45pub mod seqcell;
46pub mod snapshot;
47pub mod spec_confidence;
48
49#[cfg(test)]
50mod nvml_script;
51
52pub use hub::{DeviceState, Telemetry, global};
53pub use level::{ConfigError, Level, TelemetryConfig};