metrale_telemetry/
kernel_audit.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The kernel-resolution audit: every lookup the CUDA backend's
4//! `GpuBackend::kernel(module, func)` makes, whether it resolved, and the call
5//! site that issued it; plus the boot gate's seal.
6//!
7//! A failed optional lookup (`try_kernel` returns handle 0) raises no error by
8//! itself; the audit records it. The boot gate
9//! (`serve_phases::kernel_gate::audit_and_gate`) prints
10//! [`render_kernel_table`] and then calls [`seal`]. After the seal a failed
11//! lookup aborts the process or, with
12//! `--dangerously-allow-unresolved-kernel-lookups`, warns once per seal.
13//!
14//! Owner: telemetry.
15//! Invariants: between [`seal`] and [`unseal`], every failed lookup passed to
16//! [`record`] adds 1 to [`unresolved_lookups`] before anything else happens.
17
18mod report;
19
20pub use report::{render_kernel_table, unresolved_report};
21
22use std::collections::BTreeMap;
23use std::panic::Location;
24use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
25
26/// 2026-09-26: True from [`seal`] until [`unseal`].
27static SEALED: AtomicBool = AtomicBool::new(false);
28/// 2026-09-26: `--dangerously-allow-unresolved-kernel-lookups`, as handed to [`seal`].
29static ALLOW_UNRESOLVED: AtomicBool = AtomicBool::new(false);
30/// 2026-09-26: Unresolved lookups for the live model: the gate's count, plus
31/// every failed lookup after the seal.
32static UNRESOLVED: AtomicU64 = AtomicU64::new(0);
33/// 2026-09-26: Latch so an allowed late miss warns once per seal.
34static LATE_WARNED: AtomicBool = AtomicBool::new(false);
35
36/// 2026-09-26: One deduped kernel-resolution row.
37#[derive(Clone, Debug)]
38pub struct AuditRow {
39    pub module: String,
40    pub func: String,
41    /// 2026-09-26: True if any lookup of this `(module, func)` resolved.
42    pub loaded: bool,
43    /// 2026-09-26: Call site of the first lookup of this pair, captured through
44    /// `#[track_caller]`.
45    pub site: &'static Location<'static>,
46}
47
48impl AuditRow {
49    /// 2026-09-26: `module::func`.
50    pub fn name(&self) -> String {
51        format!("{}::{}", self.module, self.func)
52    }
53}
54
55/// 2026-09-26: Record one kernel lookup; the CUDA backend's
56/// `GpuBackend::kernel` calls it.
57///
58/// `site` is passed in, not taken with `#[track_caller]` here, because this
59/// function's caller is the backend, not the call site that asked.
60pub fn record(module: &str, func: &str, loaded: bool, site: &'static Location<'static>) {
61    if !loaded && SEALED.load(Ordering::Acquire) {
62        late_miss(module, func, site);
63    }
64    if let Ok(mut v) = crate::run_metrics::metrics().kernel_audit.lock() {
65        v.push((module.to_string(), func.to_string(), loaded, site));
66    }
67}
68
69/// 2026-09-26: A kernel lookup that failed after the seal, which the boot gate
70/// could not see. It is counted; then the process aborts, or with the allow
71/// flag warns once per seal.
72fn late_miss(module: &str, func: &str, site: &'static Location<'static>) {
73    UNRESOLVED.fetch_add(1, Ordering::Relaxed);
74    if ALLOW_UNRESOLVED.load(Ordering::Relaxed) {
75        if !LATE_WARNED.swap(true, Ordering::Relaxed) {
76            tracing::warn!(
77                "kernel lookup {module}::{func} at {}:{} failed AFTER the boot audit sealed. \
78                 This lookup is not eager, so the boot gate could not see it. Continuing \
79                 because --dangerously-allow-unresolved-kernel-lookups was passed. \
80                 Performance may be seriously degraded. We recommend you open a GitHub issue \
81                 and/or open a PR to solve this issue. \
82                 (metrale_kernel_lookups_unresolved counts every occurrence.)",
83                site.file(),
84                site.line(),
85            );
86        }
87        return;
88    }
89    // 2026-09-26: Abort, not panic: a panic would unwind only the calling
90    // thread and leave the process serving.
91    tracing::error!(
92        "kernel lookup {module}::{func} at {}:{} failed AFTER the boot audit sealed — a \
93         non-eager lookup that the boot gate could not see. The dispatch that asked for it is \
94         now on a silent fallback path. Aborting. Pass \
95         --dangerously-allow-unresolved-kernel-lookups to downgrade this to a warning.",
96        site.file(),
97        site.line(),
98    );
99    std::process::abort();
100}
101
102/// 2026-09-26: Seal the audit for this run. `unresolved` is the gate's count
103/// of failed lookups not declared expected-absent; `allow` is
104/// `--dangerously-allow-unresolved-kernel-lookups`.
105pub fn seal(unresolved: u64, allow: bool) {
106    UNRESOLVED.store(unresolved, Ordering::Relaxed);
107    ALLOW_UNRESOLVED.store(allow, Ordering::Relaxed);
108    LATE_WARNED.store(false, Ordering::Relaxed);
109    SEALED.store(true, Ordering::Release);
110}
111
112/// 2026-09-26: Re-open the audit for a new model load. Called from
113/// [`crate::run_metrics::reset_for_new_run`].
114pub fn unseal() {
115    SEALED.store(false, Ordering::Release);
116    UNRESOLVED.store(0, Ordering::Relaxed);
117    LATE_WARNED.store(false, Ordering::Relaxed);
118}
119
120/// 2026-09-26: Unresolved kernel lookups for the live model. `/metrics`
121/// exports it as `metrale_kernel_lookups_unresolved`.
122pub fn unresolved_lookups() -> u64 {
123    UNRESOLVED.load(Ordering::Relaxed)
124}
125
126/// 2026-09-26: The resolution rows: one per `(module, func)`, sorted, `loaded`
127/// true if any lookup of that pair resolved.
128pub fn audit_rows() -> Vec<AuditRow> {
129    let mut resolved: BTreeMap<(String, String), (bool, &'static Location<'static>)> =
130        BTreeMap::new();
131    if let Ok(v) = crate::run_metrics::metrics().kernel_audit.lock() {
132        for (m, f, ok, site) in v.iter() {
133            let e = resolved
134                .entry((m.clone(), f.clone()))
135                .or_insert((false, *site));
136            e.0 = e.0 || *ok;
137        }
138    }
139    resolved
140        .into_iter()
141        .map(|((module, func), (loaded, site))| AuditRow {
142            module,
143            func,
144            loaded,
145            site,
146        })
147        .collect()
148}
149
150/// 2026-09-26: The failed lookups, split by whether the operator must act.
151/// The boot gate, its log table and the TUI kernel table all split through
152/// [`split_failures`].
153#[derive(Clone, Debug, Default)]
154pub struct FailureSplit {
155    /// 2026-09-26: Not declared in `[expected_absent]`. The boot gate refuses
156    /// to serve while any is present, unless the allow flag is set.
157    pub required: Vec<AuditRow>,
158    /// 2026-09-26: Declared in this target's MODEL.toml `[expected_absent]`.
159    /// Never fatal.
160    pub expected: Vec<AuditRow>,
161}
162
163/// 2026-09-26: Split [`audit_rows`]'s failures against a target's
164/// `[expected_absent]` declaration (`TargetPtxSet::expected_absent`).
165pub fn classify_failures(expected_absent: &[(&str, &str)]) -> FailureSplit {
166    split_failures(&audit_rows(), expected_absent)
167}
168
169/// 2026-09-26: [`classify_failures`] over rows the caller already has.
170pub fn split_failures(rows: &[AuditRow], expected_absent: &[(&str, &str)]) -> FailureSplit {
171    let (expected, required): (Vec<AuditRow>, Vec<AuditRow>) =
172        rows.iter().filter(|r| !r.loaded).cloned().partition(|r| {
173            expected_absent
174                .iter()
175                .any(|(em, ef)| *em == r.module.as_str() && *ef == r.func.as_str())
176        });
177    FailureSplit { required, expected }
178}
metrale_telemetry/kernel_audit.rs

metrale_telemetry/
kernel_audit.rs
