metrale_bench/benchmarks/agentic/
preflight.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Warm-up sanity check before a tier: the endpoint must answer `4`
4//! to `2+2`, or the run stops. A port of `warmup_endpoint` in
5//! `bench/fp8_dgx2_drift/harness/run_tier.sh`. A 200 from `/v1/models` proves
6//! only that a server is listening; this proves the checkpoint still decodes.
7//!
8//! Owner: bench, agentic.
9//! Invariants: none beyond the types.
10
11use std::time::Duration;
12
13use anyhow::{Result, bail};
14use serde_json::json;
15
16use crate::http;
17use crate::plugin::PluginHandle;
18
19/// 2026-09-26: Verbatim from the harness's `warmup_endpoint` request body.
20pub const SANITY_PROMPT: &str = "What is 2+2? Respond with just the number.";
21
22/// 2026-09-26: True when either channel holds a `4` with no ASCII alphanumeric
23/// or `_` on either side. `warmup_endpoint` greps `content` and
24/// `reasoning_content` merged, with the same boundary.
25pub fn answered(text: &str, reasoning: &str) -> bool {
26    [text, reasoning].iter().any(|channel| {
27        channel.match_indices('4').any(|(index, _)| {
28            let before = channel[..index].chars().next_back();
29            let after = channel[index + 1..].chars().next();
30            let identifier = |c: char| c.is_ascii_alphanumeric() || c == '_';
31            !before.is_some_and(identifier) && !after.is_some_and(identifier)
32        })
33    })
34}
35
36/// 2026-09-26: Asks the endpoint 2+2. An error if the request fails, both
37/// channels are empty, or neither says `4` (`answered`).
38pub async fn sanity_check(handle: &PluginHandle, timeout: Duration) -> Result<()> {
39    let target = handle.target();
40    let body = json!({
41        "model": target.model,
42        "messages": [{"role": "user", "content": SANITY_PROMPT}],
43        // 2026-09-26: The harness request's 80 tokens. Thinking is off because
44        // 80 tokens is not a thinking budget.
45        "chat_template_kwargs": {"enable_thinking": false},
46        "max_tokens": 80,
47        "temperature": 0.0,
48        "stream": true,
49    });
50    let outcome = http::chat_stream(target, &body, timeout).await?;
51    if outcome.text.trim().is_empty() && outcome.reasoning.trim().is_empty() {
52        bail!(
53            "warm-up: {} returned no parseable response",
54            target.base_url
55        );
56    }
57    if !answered(&outcome.text, &outcome.reasoning) {
58        bail!(
59            "warm-up: {} did not answer '4' to 2+2 — catastrophic regression, halting: {:?}",
60            target.base_url,
61            crate::benchmarks::one_line(format!("{} {}", outcome.text, outcome.reasoning))
62        );
63    }
64    Ok(())
65}
66
67#[cfg(test)]
68mod tests {
69    use std::sync::Arc;
70    use std::sync::atomic::AtomicBool;
71
72    use tokio::io::{AsyncReadExt, AsyncWriteExt};
73    use tokio::net::TcpListener;
74
75    use super::*;
76    use crate::artifacts::ArtifactStore;
77    use crate::plugin::TargetEndpoint;
78
79    async fn reasoning_server() -> (u16, tokio::sync::oneshot::Receiver<serde_json::Value>) {
80        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
81        let port = listener.local_addr().unwrap().port();
82        let (sent, request) = tokio::sync::oneshot::channel();
83        tokio::spawn(async move {
84            let (mut socket, _) = listener.accept().await.unwrap();
85            let mut bytes = Vec::new();
86            let mut buf = [0u8; 4096];
87            let (header_end, length) = loop {
88                let n = socket.read(&mut buf).await.unwrap();
89                assert!(n > 0, "client closed before sending its request");
90                bytes.extend_from_slice(&buf[..n]);
91                let Some(header_end) = bytes.windows(4).position(|w| w == b"\r\n\r\n") else {
92                    continue;
93                };
94                let headers = String::from_utf8_lossy(&bytes[..header_end]);
95                let length = headers
96                    .lines()
97                    .find_map(|line| {
98                        line.to_ascii_lowercase()
99                            .strip_prefix("content-length: ")
100                            .and_then(|value| value.parse::<usize>().ok())
101                    })
102                    .expect("request must declare content length");
103                if bytes.len() >= header_end + 4 + length {
104                    break (header_end, length);
105                }
106            };
107            let body = serde_json::from_slice(&bytes[header_end + 4..header_end + 4 + length])
108                .expect("request body must be JSON");
109            sent.send(body).unwrap();
110
111            let body = "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"2+2 is 4\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
112            socket
113                .write_all(
114                    format!(
115                        "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
116                        body.len()
117                    )
118                    .as_bytes(),
119                )
120                .await
121                .unwrap();
122        });
123        (port, request)
124    }
125
126    #[test]
127    fn the_prompt_is_the_harness_prompt() {
128        let harness = include_str!("../../../../../bench/fp8_dgx2_drift/harness/run_tier.sh");
129        let request = harness
130            .lines()
131            .find_map(|line| {
132                line.trim_start()
133                    .strip_prefix("-d '")
134                    .and_then(|body| body.strip_suffix("' 2>&1)"))
135            })
136            .expect("run_tier.sh must carry the warm-up request as JSON");
137        let body: serde_json::Value =
138            serde_json::from_str(request).expect("the harness warm-up request must be valid JSON");
139        assert_eq!(body["messages"][0]["content"], SANITY_PROMPT);
140    }
141
142    #[test]
143    fn either_channel_may_carry_the_answer() {
144        assert!(answered("4", ""));
145        assert!(answered("", "2+2 is 4"));
146        assert!(answered("The answer is 4.", ""));
147        assert!(!answered("", ""));
148        assert!(!answered("five", "let me think"));
149        assert!(!answered("42", ""), "42 is not the answer to 2+2");
150        assert!(
151            !answered("status 404", ""),
152            "an error code is not an answer"
153        );
154        assert!(
155            !answered("4ever", ""),
156            "a digit inside a word is not an answer"
157        );
158    }
159
160    #[test]
161    fn the_harness_uses_the_same_answer_boundary() {
162        let harness = include_str!("../../../../../bench/fp8_dgx2_drift/harness/run_tier.sh");
163        assert!(
164            harness.contains("grep -Eq '(^|[^[:alnum:]_])4([^[:alnum:]_]|$)'"),
165            "the shell and Rust preflights must reject 42 and 404 alike"
166        );
167    }
168
169    #[tokio::test]
170    async fn sanity_check_sends_the_declared_request_and_reads_reasoning() {
171        let (port, request) = reasoning_server().await;
172        let (events, receiver) = std::sync::mpsc::channel();
173        let handle = PluginHandle::new(
174            1,
175            TargetEndpoint::local(port, "test-model"),
176            ArtifactStore::with_root(std::env::temp_dir().join("metrale-preflight-test")),
177            events,
178            Arc::new(AtomicBool::new(false)),
179        );
180
181        sanity_check(&handle, Duration::from_secs(2))
182            .await
183            .expect("a standalone 4 in reasoning must pass");
184        drop(receiver);
185        let body = request.await.expect("server must observe the request");
186        assert_eq!(body["model"], "test-model");
187        assert_eq!(
188            body["messages"],
189            json!([{"role": "user", "content": SANITY_PROMPT}])
190        );
191        assert_eq!(
192            body["chat_template_kwargs"],
193            json!({"enable_thinking": false})
194        );
195        assert_eq!(body["max_tokens"], 80);
196        assert_eq!(body["temperature"], 0.0);
197        assert_eq!(body["stream"], true);
198    }
199}
metrale_bench/benchmarks/agentic/preflight.rs

metrale_bench/benchmarks/agentic/
preflight.rs
