metrale_storage/
lib.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-25: metrale-storage: the KV, SSM-snapshot, expert and weight storage tiers
4//! (NVMe files, host RAM, RDMA peers), the peers that serve them, and their CUDA helpers.
5//!
6//! Owner: storage.
7//! Invariants:
8//! - The CUDA driver bindings (`cuda_min`, `cuda_module`, `cuda_graph`) exist only with
9//!   the `cuda` feature, so every module that calls the driver is gated on it too.
10//! - Without the `cuda` feature, `stubs` supplies `HighSpeedSwap`, `with_local`,
11//!   `local_installed` and `install_local` under the same names.
12
13#![deny(warnings)]
14#![deny(clippy::all)]
15
16#[cfg(feature = "cuda")]
17pub mod cuda_graph;
18#[cfg(feature = "cuda")]
19pub mod cuda_min;
20#[cfg(feature = "cuda")]
21pub mod cuda_module;
22
23#[cfg(feature = "cuda")]
24pub use cuda_module::{CudaEvent, CudaModule, launch_kernel};
25
26pub mod attention_ref;
27pub mod cascade_policy;
28pub mod config;
29pub mod eviction;
30pub mod expert;
31pub mod expert_pack;
32pub mod expert_peer;
33pub mod group;
34pub mod kv_paging;
35pub mod model_dims;
36pub mod predictor_ref;
37pub mod projection;
38// 2026-09-25: The one-sided RDMA KV backend: a `StorageBackend` that offloads and
39// restores KV groups to a `cache_peer` over verbs.
40#[cfg(all(feature = "cuda", metrale_rdma_verbs))]
41pub mod rdma_kv_backend;
42pub mod rdma_snapshot;
43pub mod snapshot_swap;
44pub mod weight_peer;
45
46// 2026-09-25: `cache_peer` serves remote-RAM arenas over one-sided RDMA; `blade_cap`
47// is the commit ledger it and `expert_peer` reserve against. The handshake that
48// reserves is compiled only with `metrale_rdma_verbs`, hence the `dead_code` allow.
49#[cfg(unix)]
50#[allow(dead_code)]
51pub(crate) mod blade_cap;
52pub mod cache_peer;
53
54// 2026-09-25: Re-exported without the `cuda` feature: metrale-model-engine's
55// `high_speed_swap_dims` returns it on every build.
56pub use model_dims::ModelDims;
57
58#[cfg(feature = "cuda")]
59pub mod layout;
60
61#[cfg(feature = "cuda")]
62pub mod backend;
63#[cfg(all(feature = "cuda", target_os = "linux"))]
64pub mod bench;
65#[cfg(feature = "cuda")]
66pub mod cascade_backend;
67#[cfg(feature = "cuda")]
68pub mod expert_arena;
69#[cfg(feature = "cuda")]
70pub mod expert_tier;
71#[cfg(feature = "cuda")]
72pub mod ngram_cache;
73#[cfg(feature = "cuda")]
74mod ngram_cache_fault;
75// 2026-09-25: RDMA expert staging, unix only.
76#[cfg(all(feature = "cuda", unix))]
77pub mod expert_tier_rdma;
78#[cfg(feature = "cuda")]
79pub mod high_speed_swap;
80#[cfg(feature = "cuda")]
81pub mod predictor;
82#[cfg(all(feature = "cuda", target_os = "linux"))]
83pub mod probe;
84#[cfg(feature = "cuda")]
85pub mod scratch_pool;
86#[cfg(feature = "cuda")]
87pub mod tiled_attention;
88#[cfg(feature = "cuda")]
89pub use backend::{PosixBackend, ReadRequest, StorageBackend};
90// 2026-09-25: io_uring is Linux-only.
91#[cfg(all(feature = "cuda", target_os = "linux"))]
92pub use backend::IoUringBackend;
93pub use config::HighSpeedSwapConfig;
94pub use eviction::EvictionPolicy;
95pub use expert::{
96    ExpertKey, ExpertLayout, ExpertRecordHeader, ExpertRecordId, ExpertRecordSpec, Proj, ProjBytes,
97};
98#[cfg(feature = "cuda")]
99pub use expert_arena::ExpertArena;
100pub use expert_pack::{ExpertFileReader, ExpertFileWriter};
101pub use expert_pack::{ExpertIndex, ProjData, ProjView, pack_record, unpack_record};
102#[cfg(feature = "cuda")]
103pub use expert_tier::{
104    ArenaSlot, ExpertResidency, ExpertTier, PosixTier, TierKind, UmaArenaTier, open_tier,
105};
106#[cfg(all(feature = "cuda", unix))]
107pub use expert_tier_rdma::RdmaTier;
108#[cfg(feature = "cuda")]
109pub use high_speed_swap::{HighSpeedSwap, install_local, local_installed, with_local};
110#[cfg(all(feature = "cuda", metrale_rdma_verbs))]
111pub use kv_paging::KvPagingBackend;
112#[cfg(feature = "cuda")]
113pub use ngram_cache::NgramRowCache;
114#[cfg(all(feature = "cuda", metrale_rdma_verbs))]
115pub use rdma_kv_backend::RdmaKvBackend;
116pub use rdma_snapshot::RdmaSnapshotArena;
117
118/// 2026-09-25: `true` when build.rs emitted `metrale_rdma_verbs` for this crate. The
119/// verbs shim is built by metrale-gpu-sys, and `rustc-cfg` does not cross crates, so
120/// build.rs re-emits the cfg from that crate's `links` metadata.
121/// `rdma_verbs_probe_tests` checks it matches `metrale_gpu_sys::verbs_enabled()`.
122pub const fn rdma_verbs_enabled() -> bool {
123    cfg!(metrale_rdma_verbs)
124}
125
126#[cfg(test)]
127mod rdma_verbs_probe_tests;
128
129// 2026-09-25: Gated on the `cuda` feature alone: `HighSpeedSwap` picks
130// `IoUringBackend` on Linux and `PosixBackend` elsewhere, so any CUDA build gets the
131// real one.
132#[cfg(not(feature = "cuda"))]
133mod stubs;
134#[cfg(not(feature = "cuda"))]
135pub use stubs::{HighSpeedSwap, install_local, local_installed, with_local};
136
137#[cfg(feature = "cuda")]
138pub use predictor::{Predictor, PredictorDims};
139#[cfg(all(feature = "cuda", target_os = "linux"))]
140pub use probe::{Backend, ProbeConfig, ProbeResult, run_probe};
141pub use projection::{PredictorShape, build_projection};
142#[cfg(feature = "cuda")]
143pub use tiled_attention::{TiledAttention, TiledAttentionDims};
144#[cfg(feature = "cuda")]
145pub use weight_peer::{WeightManifest, WeightTensorRecord};
146pub mod tier;
metrale_storage/lib.rs

metrale_storage/
lib.rs
