metrale_bench/
headless.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Driving a benchmark without a terminal: start it on the
4//! executor, drain its messages until it finishes, build the run record and
5//! save it with [`crate::history::save`], as the dashboard also does.
6//!
7//! Owner: bench.
8//! Invariants:
9//! - [`run_blocking`] validates the parameters before the executor starts.
10//! - A run that ends without a terminal frame is recorded as a failure.
11
12use std::path::PathBuf;
13use std::time::Duration;
14
15use anyhow::Result;
16
17use crate::benchmark::BenchmarkDescriptor;
18use crate::coherence::CoherencePolicy;
19use crate::executor::{BenchmarkExecutor, ExecutorMessage};
20use crate::history::{self, RunRecord, RunSource};
21use crate::params::ParamValues;
22use crate::plugin::{PluginEvent, TargetEndpoint};
23use crate::result::{BenchmarkResult, VerdictKind};
24
25/// 2026-09-26: How the driver behaves, as opposed to what it runs.
26#[derive(Clone, Debug)]
27pub struct HeadlessOptions {
28    /// 2026-09-26: Sleep between drains, and so between cancellation checks.
29    pub poll: Duration,
30    /// 2026-09-26: Save the run into the executor's artifact store, under
31    /// `runs/<benchmark>`.
32    pub save: bool,
33    pub source: RunSource,
34    /// 2026-09-26: Recorded on the run so a result can be traced to the build.
35    pub metrale_version: String,
36    /// 2026-09-26: Whether the executor probes the endpoint's coherence before
37    /// measuring.
38    pub coherence: CoherencePolicy,
39    /// 2026-09-26: The box class's temperature ceilings for the hardware
40    /// pre-check (`hardware::limits`), when the caller could read them.
41    pub temp_ceilings: Option<crate::hardware::policy::TempCeilings>,
42}
43
44impl HeadlessOptions {
45    pub fn cli(metrale_version: impl Into<String>) -> Self {
46        Self {
47            poll: Duration::from_millis(250),
48            save: true,
49            source: RunSource::Cli,
50            metrale_version: metrale_version.into(),
51            coherence: CoherencePolicy::Probe,
52            temp_ceilings: None,
53        }
54    }
55}
56
57/// 2026-09-26: What to run, and against what.
58pub struct RunRequest {
59    pub descriptor: &'static BenchmarkDescriptor,
60    pub values: ParamValues,
61    /// 2026-09-26: Where to measure. Its `serve_overrides` are copied into
62    /// the run record.
63    pub target: TargetEndpoint,
64    pub options: HeadlessOptions,
65}
66
67/// 2026-09-26: Callbacks for a run as it happens. Every method defaults to
68/// nothing.
69pub trait RunReporter {
70    fn started(&mut self, _request: &RunRequest) {}
71    fn event(&mut self, _event: &PluginEvent) {}
72    fn frame(&mut self, _frame: &BenchmarkResult) {}
73}
74
75/// 2026-09-26: A reporter that says nothing: the CLI's JSON output format
76/// and the tests use it.
77pub struct SilentReporter;
78impl RunReporter for SilentReporter {}
79
80#[derive(Debug)]
81pub struct RunOutcome {
82    pub record: RunRecord,
83    /// 2026-09-26: Where it was written, when `options.save`.
84    pub saved_to: Option<PathBuf>,
85    pub cancelled: bool,
86}
87
88impl RunOutcome {
89    /// 2026-09-26: `1` when the run was cancelled or did not complete, `2` when
90    /// it completed with a failing verdict, else `0`.
91    pub fn exit_code(&self) -> i32 {
92        if self.cancelled
93            || !matches!(
94                self.record.frame.status,
95                crate::result::RunStatus::Completed
96            )
97        {
98            return 1;
99        }
100        match self.record.verdict_kind() {
101            Some(VerdictKind::Fail) => 2,
102            _ => 0,
103        }
104    }
105}
106
107/// 2026-09-26: Run to completion, pumping the executor's channels. Blocks the
108/// calling thread; from async, use `spawn_blocking`.
109///
110/// `should_cancel` is polled once per drain; the first true cancels the run.
111/// Errors on invalid parameters, before the run starts, and when saving
112/// fails.
113pub fn run_blocking(
114    executor: &BenchmarkExecutor,
115    request: RunRequest,
116    reporter: &mut dyn RunReporter,
117    should_cancel: &dyn Fn() -> bool,
118) -> Result<RunOutcome> {
119    let specs = request.descriptor.build().parameters();
120    request.values.validate_against(&specs)?;
121
122    reporter.started(&request);
123    let run = executor.start(
124        request.descriptor,
125        request.values.clone(),
126        request.target.clone(),
127        request.options.coherence,
128        request.options.temp_ceilings,
129    );
130
131    let mut terminal: Option<BenchmarkResult> = None;
132    loop {
133        for message in run.drain() {
134            dispatch(message, reporter, &mut terminal);
135        }
136        if should_cancel() && !run.is_cancelled() {
137            run.cancel();
138            reporter.event(&PluginEvent::Status(
139                "cancelling — stopping after the request in flight".into(),
140            ));
141        }
142        if run.is_finished() {
143            break;
144        }
145        std::thread::sleep(request.options.poll);
146    }
147
148    // 2026-09-26: The executor sets `finished` in `teardown`, after its last
149    // send, so drain until a drain comes back empty.
150    loop {
151        let messages = run.drain();
152        if messages.is_empty() {
153            break;
154        }
155        for message in messages {
156            dispatch(message, reporter, &mut terminal);
157        }
158        std::thread::sleep(Duration::from_millis(10));
159    }
160
161    let cancelled = run.is_cancelled();
162    let frame = terminal.unwrap_or_else(|| {
163        BenchmarkResult::failed(
164            "run",
165            "the run ended without a terminal frame",
166            Duration::ZERO,
167        )
168    });
169
170    let mut record = RunRecord::new(
171        request.descriptor,
172        &request.values,
173        &request.target,
174        request.target.serve_overrides.clone(),
175        request.options.source,
176        &request.options.metrale_version,
177        frame,
178    );
179    let saved_to = if request.options.save {
180        Some(history::save(executor.artifacts(), &mut record)?)
181    } else {
182        None
183    };
184
185    Ok(RunOutcome {
186        record,
187        saved_to,
188        cancelled,
189    })
190}
191
192fn dispatch(
193    message: ExecutorMessage,
194    reporter: &mut dyn RunReporter,
195    terminal: &mut Option<BenchmarkResult>,
196) {
197    match message {
198        ExecutorMessage::Event(e) => reporter.event(&e),
199        ExecutorMessage::Frame(f) => {
200            reporter.frame(&f);
201            if f.status.is_terminal() {
202                *terminal = Some(*f);
203            }
204        }
205    }
206}
207
208#[cfg(test)]
209#[path = "headless_tests.rs"]
210mod tests;
metrale_bench/headless.rs

metrale_bench/
headless.rs
