metrale_gpu_runtime/cuda_backend/
redzone.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-25: Red-zone diagnostic: an opt-in guard band after each `alloc`, the
4//! `METRALE_REDZONE*` variables that configure it, and the passes that
5//! re-fill and check the bands.
6//!
7//! Owner: gpu-runtime (CUDA backend).
8//! Invariants:
9//! - Each `METRALE_REDZONE*` variable this module reads is read once per
10//!   process (one `OnceLock` each).
11
12use super::*;
13
14/// 2026-09-25: One allocation's trailing guard band: `pad_bytes` after
15/// `user_ptr + user_bytes`, for the allocation with creation index `idx`.
16#[derive(Clone, Copy)]
17pub(crate) struct RedZone {
18    user_ptr: u64,
19    user_bytes: usize,
20    pad_bytes: usize,
21    idx: usize,
22}
23
24/// 2026-09-25: `METRALE_REDZONE=<bytes>`: guard-band size, rounded up to a
25/// multiple of 16. Unset, unparsable or 0 disables the red zones.
26pub(crate) fn redzone_bytes() -> usize {
27    static N: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
28    *N.get_or_init(|| {
29        std::env::var("METRALE_REDZONE")
30            .ok()
31            .and_then(|v| v.parse::<usize>().ok())
32            .map(|n| if n == 0 { 0 } else { n.next_multiple_of(16) })
33            .unwrap_or(0)
34    })
35}
36
37/// 2026-09-25: `METRALE_REDZONE_MIN_IDX=<n>`: pad only allocations whose
38/// creation index ([`ALLOC_SEQ`]) is `>= n`. Unset or unparsable is 0: every
39/// allocation.
40pub(crate) fn redzone_min_idx() -> usize {
41    static N: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
42    *N.get_or_init(|| {
43        std::env::var("METRALE_REDZONE_MIN_IDX")
44            .ok()
45            .and_then(|v| v.parse::<usize>().ok())
46            .unwrap_or(0)
47    })
48}
49
50/// 2026-09-25: `METRALE_REDZONE_TRACE_IDX=<n>`: log a backtrace when the
51/// padded allocation with creation index `n` is made.
52pub(crate) fn redzone_trace_idx() -> Option<usize> {
53    static N: std::sync::OnceLock<Option<usize>> = std::sync::OnceLock::new();
54    *N.get_or_init(|| {
55        std::env::var("METRALE_REDZONE_TRACE_IDX")
56            .ok()
57            .and_then(|v| v.parse::<usize>().ok())
58    })
59}
60
61/// 2026-09-25: Creation index source: incremented once per
62/// `MetraleCudaBackend::alloc` call, failed calls included, across every
63/// backend in the process.
64pub(crate) static ALLOC_SEQ: std::sync::atomic::AtomicUsize =
65    std::sync::atomic::AtomicUsize::new(0);
66
67/// 2026-09-25: `METRALE_REDZONE_FILL=<decimal byte>`: the byte written into
68/// new guard bands and expected by `scan_redzones`. Unset or unparsable is
69/// `0xEE`.
70///
71/// A configurable fill separates the two kinds of overrun: a write past the
72/// end changes the band and `scan_redzones` reports it, while a read past the
73/// end leaves the band intact but lets the fill reach the computation, so two
74/// runs with different fills can produce different tokens.
75pub(crate) fn redzone_fill() -> u8 {
76    static F: std::sync::OnceLock<u8> = std::sync::OnceLock::new();
77    *F.get_or_init(|| {
78        std::env::var("METRALE_REDZONE_FILL")
79            .ok()
80            .and_then(|v| v.parse::<u8>().ok())
81            .unwrap_or(0xEE)
82    })
83}
84
85impl MetraleCudaBackend {
86    /// 2026-09-25: Register one allocation's guard band.
87    pub(crate) fn record_redzone(
88        &self,
89        user_ptr: u64,
90        user_bytes: usize,
91        pad_bytes: usize,
92        idx: usize,
93    ) {
94        self.redzones.lock().push(RedZone {
95            user_ptr,
96            user_bytes,
97            pad_bytes,
98            idx,
99        });
100    }
101
102    pub(crate) fn forget_redzone(&self, user_ptr: u64) {
103        self.redzones.lock().retain(|z| z.user_ptr != user_ptr);
104    }
105
106    /// 2026-09-25: Re-fill every guard band: `0xEE` for zones whose creation
107    /// index is in `[lo, hi)`, `0x00` for all the others. Only guard-band
108    /// bytes are written.
109    ///
110    /// For bisecting a read past the end: narrowing `[lo, hi)` to the range
111    /// whose fill changes the output finds the allocation being read past.
112    pub fn poison_redzones(&self, lo: usize, hi: usize) -> anyhow::Result<()> {
113        let zones: Vec<RedZone> = self.redzones.lock().clone();
114        for z in &zones {
115            let v = if z.idx >= lo && z.idx < hi {
116                0xEEu8
117            } else {
118                0x00u8
119            };
120            let st = unsafe { cuMemsetD8_v2(z.user_ptr + z.user_bytes as u64, v, z.pad_bytes) };
121            if st != 0 {
122                anyhow::bail!("poison_redzones: cuMemsetD8_v2 failed: status {st}");
123            }
124        }
125        Ok(())
126    }
127
128    /// 2026-09-25: Read every guard band back and count the ones holding a
129    /// byte other than `redzone_fill()`. Each is logged at ERROR with its
130    /// creation index, size, the first and last changed pad offsets, how many
131    /// pad bytes changed, and the first changed value, and is then re-filled,
132    /// so a later scan reports only new writes.
133    pub fn scan_redzones(&self) -> anyhow::Result<usize> {
134        let fill = redzone_fill();
135        let zones: Vec<RedZone> = self.redzones.lock().clone();
136        let mut bad = 0usize;
137        let mut host = Vec::new();
138        for z in &zones {
139            host.clear();
140            host.resize(z.pad_bytes, 0u8);
141            let st = unsafe {
142                cuMemcpyDtoH_v2(
143                    host.as_mut_ptr() as *mut std::ffi::c_void,
144                    z.user_ptr + z.user_bytes as u64,
145                    z.pad_bytes,
146                )
147            };
148            if st != 0 {
149                anyhow::bail!("redzone scan: cuMemcpyDtoH_v2 failed: status {st}");
150            }
151            let Some(first) = host.iter().position(|b| *b != fill) else {
152                continue;
153            };
154            let changed = host.iter().filter(|b| **b != fill).count();
155            let last = host.iter().rposition(|b| *b != fill).unwrap_or(first);
156            tracing::error!(
157                "🔴 REDZONE VIOLATION alloc#{} user_bytes={} pad={} :                  bytes [{}..={}] past the end were written ({} of {} pad bytes changed),                  first bad value {:#04x}",
158                z.idx,
159                z.user_bytes,
160                z.pad_bytes,
161                first,
162                last,
163                changed,
164                z.pad_bytes,
165                host[first],
166            );
167            bad += 1;
168            let st = unsafe { cuMemsetD8_v2(z.user_ptr + z.user_bytes as u64, fill, z.pad_bytes) };
169            if st != 0 {
170                anyhow::bail!("redzone scan: refill cuMemsetD8_v2 failed: status {st}");
171            }
172        }
173        tracing::info!(
174            "redzone scan: {} zones checked, {} violated",
175            zones.len(),
176            bad
177        );
178        Ok(bad)
179    }
180}
metrale_gpu_runtime/cuda_backend/redzone.rs

metrale_gpu_runtime/cuda_backend/
redzone.rs
