metrale_core/
fault.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-25: Process-wide latch for a destroyed CUDA context, the probe-based verdict that sets it, and the exit status it maps to.
4//!
5//! [`classify`] decides from a probe result, not from the failing call's error
6//! code or text: the gpu-runtime fault probe (`cuda_backend/fault_probe.rs`)
7//! passes the result of `cuStreamSynchronize` on the default stream, issued
8//! after the failure. The latch is read by the request middleware
9//! (`gpu_fault_middleware`), `/health` and `/health/live`, the scheduler, and
10//! `main`'s exit path.
11//!
12//! Owner: core.
13//! Invariants:
14//! - A [`FaultLatch`] is set at most once, and keeps the first reason.
15//! - `is_faulted()` is true exactly when `fault()` is `Some`: both read the
16//!   same `OnceLock<String>`, so no reader sees one without the other.
17
18use std::sync::OnceLock;
19
20/// 2026-09-25: The verdict for one failed GPU operation.
21#[derive(Debug, Clone, PartialEq, Eq)]
22pub enum Fatality {
23    /// 2026-09-25: The probe failed too: the context is unusable. The payload
24    /// is the operator-facing reason.
25    ContextLost(String),
26    /// 2026-09-25: The probe succeeded: the context still works.
27    Isolated,
28}
29
30/// 2026-09-25: Decide whether a failed GPU operation destroyed the context.
31///
32/// `probe` is the result of a call issued after the failure that succeeds on a
33/// healthy context. Only `probe` decides the verdict; `op` and `err` appear
34/// only in the reason text.
35pub fn classify(op: &str, err: &str, probe: Result<(), String>) -> Fatality {
36    match probe {
37        Ok(()) => Fatality::Isolated,
38        Err(probe_err) => Fatality::ContextLost(format!(
39            "{op} failed ({err}), and a no-op synchronize issued afterwards \
40             also failed ({probe_err}) — the CUDA context is destroyed. Errors \
41             of this class are sticky: every later driver call in this process \
42             returns the same status, so no request can be served."
43        )),
44    }
45}
46
47/// 2026-09-25: A one-shot, first-writer-wins fault flag.
48///
49/// Constructible on its own so tests need not latch the process-wide instance,
50/// which cannot be reset.
51#[derive(Debug, Default)]
52pub struct FaultLatch {
53    reason: OnceLock<String>,
54}
55
56impl FaultLatch {
57    pub const fn new() -> Self {
58        Self {
59            reason: OnceLock::new(),
60        }
61    }
62
63    /// 2026-09-25: Record a fatal fault. Returns `true` only for the call that
64    /// set the latch; later calls leave the first reason in place.
65    pub fn latch(&self, reason: impl Into<String>) -> bool {
66        self.reason.set(reason.into()).is_ok()
67    }
68
69    /// 2026-09-25: The reason for the fault, or `None` if healthy.
70    pub fn fault(&self) -> Option<&str> {
71        self.reason.get().map(String::as_str)
72    }
73
74    pub fn is_faulted(&self) -> bool {
75        self.reason.get().is_some()
76    }
77}
78
79static GLOBAL: FaultLatch = FaultLatch::new();
80
81/// 2026-09-25: The process-wide latch that the fault probe sets and the
82/// server reads.
83pub fn global() -> &'static FaultLatch {
84    &GLOBAL
85}
86
87/// 2026-09-25: Exit status of a process whose CUDA context was lost: nonzero,
88/// and distinct from the status 1 of an ordinary failure ([`exit_code`]).
89pub const EXIT_GPU_FAULT: i32 = 70;
90
91/// 2026-09-25: The process exit status for a run that is ending: a latched
92/// fault gives [`EXIT_GPU_FAULT`] whatever the run's own result, otherwise 0
93/// for success and 1 for failure. `main` exits through this, so a run that
94/// ended after a fault does not exit 0.
95pub fn exit_code(run_succeeded: bool, fault: Option<&str>) -> i32 {
96    match (run_succeeded, fault) {
97        (_, Some(_)) => EXIT_GPU_FAULT,
98        (true, None) => 0,
99        (false, None) => 1,
100    }
101}
102
103#[cfg(test)]
104#[path = "fault_tests.rs"]
105mod tests;
metrale_core/fault.rs

metrale_core/
fault.rs
