metrale_bench/benchmarks/serve_matrix/
host.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The seam the serve matrix boots checkpoints through: the serving
4//! process installs a [`ServeHost`], because this crate starts no server.
5//!
6//! Owner: bench, serve matrix.
7//! Invariants: none beyond the types.
8
9use std::sync::Arc;
10
11use anyhow::Result;
12use futures::future::BoxFuture;
13use parking_lot::RwLock;
14
15use crate::plugin::TargetEndpoint;
16
17/// 2026-09-26: Why a cached checkpoint cannot take part in the matrix. An absent
18/// checkpoint is skipped and never booted; a planned one that fails to boot
19/// fails its round.
20#[derive(Clone, Copy, Debug, PartialEq, Eq)]
21pub enum Absence {
22    /// 2026-09-26: A cache entry exists but its weights are not on disk.
23    NoWeights,
24    /// 2026-09-26: Downloaded, but its `config.json` could not be read or
25    /// parsed, so the architecture is unknown. Kept apart from
26    /// [`Absence::NoKernels`], which names an architecture.
27    NoConfig,
28    /// 2026-09-26: Downloaded and parsed, and this build compiled no kernels for it.
29    NoKernels,
30}
31
32impl Absence {
33    pub fn reason(self) -> &'static str {
34        match self {
35            Absence::NoWeights => "weights not fully downloaded",
36            Absence::NoConfig => "config.json unreadable — architecture unknown",
37            Absence::NoKernels => "no compiled kernels for this architecture",
38        }
39    }
40}
41
42/// 2026-09-26: One checkpoint the box knows about. Two quants of a model are two
43/// candidates, and so two rounds.
44#[derive(Clone, Debug, PartialEq, Eq)]
45pub struct ServeCandidate {
46    /// 2026-09-26: HF id, `org/name`.
47    pub model: String,
48    /// 2026-09-26: Quantization as the host reports it; the plan uses it in the
49    /// round label and order.
50    pub quant: String,
51    /// 2026-09-26: `None` means the round is planned.
52    pub absent: Option<Absence>,
53}
54
55impl ServeCandidate {
56    pub fn ready(model: impl Into<String>, quant: impl Into<String>) -> Self {
57        Self {
58            model: model.into(),
59            quant: quant.into(),
60            absent: None,
61        }
62    }
63
64    pub fn absent(model: impl Into<String>, quant: impl Into<String>, why: Absence) -> Self {
65        Self {
66            model: model.into(),
67            quant: quant.into(),
68            absent: Some(why),
69        }
70    }
71}
72
73/// 2026-09-26: How one round is served.
74#[derive(Clone, Copy, Debug, PartialEq, Eq)]
75pub struct ServeOptions {
76    pub max_seq_len: usize,
77    /// 2026-09-26: Serve with speculative decoding; the `speculative` parameter,
78    /// off by default.
79    pub speculative: bool,
80}
81
82/// 2026-09-26: Boots checkpoints and puts the previous one back; implemented by
83/// the serving process.
84pub trait ServeHost: Send + Sync {
85    /// 2026-09-26: Every checkpoint the box holds, each marked servable or absent.
86    fn roster(&self) -> Result<Vec<ServeCandidate>>;
87
88    /// 2026-09-26: Bring `model` up and return the endpoint once it answers.
89    fn serve(&self, model: &str, opts: ServeOptions) -> BoxFuture<'_, Result<TargetEndpoint>>;
90
91    /// 2026-09-26: Put back whatever was serving before the matrix started.
92    /// Called from the benchmark's `cleanup` once a plan exists.
93    fn restore(&self) -> BoxFuture<'_, Result<()>>;
94}
95
96/// 2026-09-26: The installed host. A static because the registry builds a
97/// benchmark from `BenchmarkDescriptor::ctor`, a `fn()` that takes no handle.
98static HOST: RwLock<Option<Arc<dyn ServeHost>>> = RwLock::new(None);
99
100/// 2026-09-26: The serving process calls this as it starts; a later call
101/// replaces the earlier host.
102pub fn install(host: Arc<dyn ServeHost>) {
103    *HOST.write() = Some(host);
104}
105
106pub fn installed() -> Option<Arc<dyn ServeHost>> {
107    HOST.read().clone()
108}
109
110/// 2026-09-26: What to tell the operator when nothing is installed; `load`
111/// fails with it.
112pub const NO_HOST: &str = "the serve matrix needs the Metrale Engine server that hosts this dashboard: it \
113                           boots each checkpoint in-process. Run it from `met serve`'s \
114                           dashboard rather than a standalone harness.";
115
116#[cfg(test)]
117mod tests {
118    use super::*;
119
120    #[test]
121    fn every_absence_reads_differently_because_each_needs_a_different_fix() {
122        assert_eq!(
123            [
124                (Absence::NoWeights, Absence::NoWeights.reason()),
125                (Absence::NoConfig, Absence::NoConfig.reason()),
126                (Absence::NoKernels, Absence::NoKernels.reason()),
127            ],
128            [
129                (Absence::NoWeights, "weights not fully downloaded"),
130                (
131                    Absence::NoConfig,
132                    "config.json unreadable — architecture unknown",
133                ),
134                (
135                    Absence::NoKernels,
136                    "no compiled kernels for this architecture",
137                ),
138            ]
139        );
140    }
141
142    #[test]
143    fn a_ready_candidate_is_distinguishable_from_an_absent_one() {
144        assert_eq!(
145            ServeCandidate::ready("org/ready", "nvfp4"),
146            ServeCandidate {
147                model: "org/ready".into(),
148                quant: "nvfp4".into(),
149                absent: None,
150            }
151        );
152        assert_eq!(
153            ServeCandidate::absent("org/absent", "fp8", Absence::NoWeights),
154            ServeCandidate {
155                model: "org/absent".into(),
156                quant: "fp8".into(),
157                absent: Some(Absence::NoWeights),
158            }
159        );
160    }
161}
metrale_bench/benchmarks/serve_matrix/host.rs

metrale_bench/benchmarks/serve_matrix/
host.rs
