metrale_bench/benchmarks/default_tier_boot/
mod.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-10-01: Default-Tier Boot: a boot-only check of the serve's memory plan. Once the
4//! serve is up it sends one short warmup wave, then reads `GET /memory` and reports the KV
5//! pool's block count and whether the serve's device footprint stays inside its
6//! `--gpu-memory-utilization` budget.
7//!
8//! Every other gate pins a reduced-precision serve; this one exists for the serve's default
9//! precision tier, whose KV pool can shrink without any of them noticing. It lands
10//! informational: the metrics are recorded and the verdict is `info` until bounds are declared
11//! for it in BENCH.toml.
12//!
13//! The pins (the benchmark's definition, not parameters):
14//! - Warmup: `WARMUP_REQUESTS` concurrent chat requests, each a distinct prompt of about
15//!   `WARMUP_PROMPT_TOKENS` tokens with `max_tokens` = `WARMUP_OUTPUT_TOKENS`, temperature 0,
16//!   seed 0. It brings the serve to the state a first real wave leaves it in.
17//! - Read: one `GET /memory` after the wave has finished.
18//!
19//! Owner: bench, default-tier-boot.
20//! Invariants: the run verdict is never a pass (`memory::verdict_for`).
21
22use crate::hardware::Sensitivity;
23use std::collections::BTreeMap;
24use std::future::Future;
25use std::time::{Duration, Instant};
26
27use anyhow::{Context, Result, bail};
28use serde_json::json;
29
30use crate::benchmark::{Benchmark, BenchmarkDescriptor, ModelExpectation};
31use crate::benchmarks::stats::{self, PromptMode};
32use crate::http;
33use crate::metadata::PluginMetadata;
34use crate::params::{ParamKind, ParamSpec, ParamValue, ParamValues};
35use crate::plugin::{Plugin, PluginHandle};
36use crate::result::{BenchmarkResult, CellStyle, LogLine, RunStatus, Stat};
37
38pub mod memory;
39pub use memory::{Footprint, MemoryReport, footprint, verdict_for};
40
41/// 2026-10-01: Concurrent warmup requests.
42const WARMUP_REQUESTS: usize = 8;
43/// 2026-10-01: Approximate prompt length of each warmup request (`stats::make_prompt`).
44const WARMUP_PROMPT_TOKENS: usize = 128;
45/// 2026-10-01: `max_tokens` of each warmup request.
46const WARMUP_OUTPUT_TOKENS: usize = 128;
47/// 2026-10-01: The server's memory endpoint.
48pub const MEMORY_PATH: &str = "/memory";
49
50const SUMMARY: &str = "Boot check of the default precision tier: KV blocks and device \
51                       footprint against the memory budget after one warmup wave";
52pub const METADATA: PluginMetadata = PluginMetadata::metrale(SUMMARY);
53
54pub const DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
55    id: "default-tier-boot",
56    name: "Default-Tier Boot Check",
57    summary: SUMMARY,
58    detail: "Boots the serve, sends one warmup wave of 8 concurrent chat requests (~128 prompt \
59             tokens, 128 output tokens each, temperature 0, seed 0), then reads GET /memory. \
60             Reports kv_blocks (the main KV pool), max_batch_size (the slots the model was \
61             built with), budget_mib (total device memory x --gpu-memory-utilization), \
62             device_footprint_mib (host MemAvailable at process start minus now, minus the \
63             server's RssAnon: on unified memory, what the serve holds on the device) and \
64             footprint_over_budget_mib (negative = within budget). Informational: no bounds \
65             are declared yet, so the verdict is info. Needs a server that answers /memory \
66             with host readings (Linux); a null reading makes the run INCONCLUSIVE.",
67    duration_hint: "~2 min",
68    expected_secs: 120,
69    updated: "2026-10-01",
70    intended_for: Some(ModelExpectation {
71        families: &["qwen3.8-27b"],
72        note: "The check is declared for unsloth/Qwen3.8-27B-NVFP4 served at its default \
73               precision tier. Other checkpoints report their own memory plan, with nothing \
74               to compare it against.",
75    }),
76    threshold_params: &[],
77    needs_confirmation: false,
78    // 2026-10-01: Block counts and byte figures; a throttled box reports the same ones.
79    sensitivity: Sensitivity::Correctness,
80    ctor: || Box::new(DefaultTierBoot::default()),
81};
82
83#[derive(Default)]
84pub struct DefaultTierBoot {
85    handle: Option<PluginHandle>,
86    timeout: Duration,
87    started: Option<Instant>,
88    probed: bool,
89    warmed: bool,
90}
91
92impl DefaultTierBoot {
93    fn handle(&self) -> Result<&PluginHandle> {
94        self.handle.as_ref().context("benchmark was not loaded")
95    }
96
97    fn elapsed(&self) -> Duration {
98        self.started.map(|s| s.elapsed()).unwrap_or_default()
99    }
100
101    fn request_body(model: &str, prompt: String) -> serde_json::Value {
102        json!({
103            "model": model,
104            "stream": true,
105            "temperature": 0.0,
106            "seed": 0,
107            "max_tokens": WARMUP_OUTPUT_TOKENS,
108            "messages": [{"role": "user", "content": prompt}],
109        })
110    }
111
112    /// 2026-10-01: The warmup wave, all requests in flight at once. Each prompt carries its own
113    /// tag, so no two share a cached prefix. Any failed request fails the run: a serve that
114    /// cannot answer a small wave has no memory plan worth reporting.
115    async fn warmup(&self) -> Result<usize> {
116        let handle = self.handle()?;
117        let target = handle.target();
118        let bodies: Vec<serde_json::Value> = (0..WARMUP_REQUESTS)
119            .map(|i| {
120                let tag = super::unique_prefix_tag(&format!("boot-{i}"), handle.run_id());
121                let prompt = stats::make_prompt(WARMUP_PROMPT_TOKENS, PromptMode::Count, &tag);
122                Self::request_body(&target.model, prompt)
123            })
124            .collect();
125        let outcomes = futures::future::join_all(
126            bodies
127                .iter()
128                .map(|body| http::chat_stream(target, body, self.timeout)),
129        )
130        .await;
131        let mut tokens = 0;
132        for (i, outcome) in outcomes.into_iter().enumerate() {
133            let outcome = outcome.with_context(|| format!("warmup request {}", i + 1))?;
134            tokens += outcome.completion_tokens;
135        }
136        Ok(tokens)
137    }
138}
139
140impl Plugin for DefaultTierBoot {
141    fn metadata(&self) -> &'static PluginMetadata {
142        &METADATA
143    }
144
145    fn load(&mut self, handle: PluginHandle) -> impl Future<Output = Result<()>> + Send {
146        self.handle = Some(handle);
147        self.started = Some(Instant::now());
148        async { Ok(()) }
149    }
150}
151
152impl Benchmark for DefaultTierBoot {
153    fn descriptor(&self) -> &'static BenchmarkDescriptor {
154        &DESCRIPTOR
155    }
156
157    fn parameters(&self) -> Vec<ParamSpec> {
158        vec![ParamSpec::new(
159            "request_timeout_s",
160            "Request timeout",
161            "Seconds before a single warmup request, or the /memory read, is abandoned. \
162             Transport-side only.",
163            ParamKind::Int { min: 30, max: 3600 },
164            ParamValue::Int(300),
165        )]
166    }
167
168    fn configure(&mut self, values: &ParamValues) -> Result<()> {
169        let specs = self.parameters();
170        values.validate_against(&specs)?;
171        self.timeout = Duration::from_secs(values.usize("request_timeout_s")? as u64);
172        self.probed = false;
173        self.warmed = false;
174        Ok(())
175    }
176
177    async fn next(&mut self) -> Result<BenchmarkResult> {
178        let handle = self.handle()?.clone();
179        handle.check_cancelled()?;
180
181        if !self.probed {
182            self.probed = true;
183            http::probe(handle.target(), Duration::from_secs(10))
184                .await
185                .context("endpoint probe failed — check the target URL and port")?;
186            return Ok(BenchmarkResult::running("probe", self.elapsed())
187                .with_progress(0, 2)
188                .log_line(LogLine::info(format!(
189                    "{} · warmup {WARMUP_REQUESTS} x (~{WARMUP_PROMPT_TOKENS} prompt, \
190                     {WARMUP_OUTPUT_TOKENS} output) tokens · then GET {MEMORY_PATH}",
191                    handle.target().base_url
192                ))));
193        }
194
195        if !self.warmed {
196            self.warmed = true;
197            handle.status(format!("warmup: {WARMUP_REQUESTS} concurrent requests"));
198            let tokens = self.warmup().await?;
199            if tokens == 0 {
200                bail!("the warmup wave produced no output token — the serve is not decoding");
201            }
202            handle.progress(1, 2);
203            return Ok(BenchmarkResult::running("warmup", self.elapsed())
204                .with_progress(1, 2)
205                .log_line(LogLine::info(format!(
206                    "warmup: {WARMUP_REQUESTS} requests, {tokens} output tokens"
207                ))));
208        }
209
210        let doc = http::get_json(handle.target(), MEMORY_PATH, self.timeout)
211            .await
212            .with_context(|| format!("reading {MEMORY_PATH}"))?;
213        let report: MemoryReport = serde_json::from_value(doc)
214            .with_context(|| format!("parsing {MEMORY_PATH}: not a MemoryReport"))?;
215        let derived = footprint(&report);
216        let verdict = verdict_for(&derived);
217        let mut metrics = BTreeMap::new();
218        let summary = match &derived {
219            Err(_) => Vec::new(),
220            Ok(f) => {
221                f.metrics(&mut metrics);
222                vec![
223                    Stat::new("KV blocks", f.kv_blocks.to_string(), "")
224                        .with_style(CellStyle::Accent),
225                    Stat::new("Slots", f.max_batch_size.to_string(), ""),
226                    Stat::new(
227                        "Device footprint",
228                        format!("{:.0}", f.device_footprint_mib),
229                        "MiB",
230                    ),
231                    Stat::new("Budget", format!("{:.0}", f.budget_mib), "MiB"),
232                    Stat::new(
233                        "Over budget",
234                        format!("{:+.0}", f.footprint_over_budget_mib),
235                        "MiB",
236                    ),
237                ]
238            }
239        };
240        handle.progress(2, 2);
241        Ok(BenchmarkResult {
242            status: RunStatus::Completed,
243            ..BenchmarkResult::running("done", self.elapsed())
244        }
245        .with_progress(2, 2)
246        .with_summary(summary)
247        .with_metrics(metrics)
248        .with_verdict(verdict)
249        .log_line(LogLine::info(format!(
250            "{MEMORY_PATH}: {} blocks x {} bytes, ledger {}",
251            report.kv_blocks,
252            report.kv_block_bytes,
253            report
254                .ledger_live_bytes
255                .map(|b| format!("{b} bytes"))
256                .unwrap_or_else(|| "null".into()),
257        ))))
258    }
259}
260
261#[cfg(test)]
262#[path = "default_tier_boot_tests.rs"]
263mod tests;
metrale_bench/benchmarks/default_tier_boot/mod.rs

metrale_bench/benchmarks/default_tier_boot/
mod.rs
