metrale_bench/benchmarks/serve_matrix/
probes.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The serve matrix's per-round probes: each turns its requests into
4//! a [`Signal`] (or a tok/s figure); the bars that read them are in `score.rs`.
5//!
6//! Owner: bench, serve matrix.
7//! Invariants:
8//! - Every probe request is greedy (temperature 0) and carries no sampling
9//!   penalty (`base_body`, and `coherence::probe` for the coherence leg). A
10//!   repetition penalty would penalise the repeated `\n` that `code_shape`
11//!   requires.
12
13use std::time::Duration;
14
15use anyhow::Result;
16use serde_json::{Value, json};
17
18use super::score::Signal;
19use crate::benchmarks::{one_line, stats};
20use crate::coherence;
21use crate::http;
22use crate::plugin::TargetEndpoint;
23
24/// 2026-09-26: Needle for the long-context probe; a test asserts the filler never
25/// contains it.
26pub const NEEDLE: &str = "PURPLE-DOLPHIN-42";
27
28/// 2026-09-26: The body every probe here starts from: streamed, greedy, no
29/// penalties, with `max_tokens`. Callers add the messages and any tools.
30fn base_body(model: &str, max_tokens: usize) -> Value {
31    json!({
32        "model": model,
33        "stream": true,
34        "temperature": 0.0,
35        "max_tokens": max_tokens,
36    })
37}
38
39fn with_user(mut body: Value, prompt: String) -> Value {
40    body["messages"] = json!([{"role": "user", "content": prompt}]);
41    body
42}
43
44/// 2026-09-26: What one round's coherence leg established.
45pub struct Coherence {
46    pub passed: usize,
47    pub total: usize,
48    /// 2026-09-26: The round's identity bar. It fails when a readable
49    /// `/v1/models` does not list this round's model, or when a coherence
50    /// request failed. An unreadable model list does not fail it
51    /// (`coherence::probe_for`).
52    pub identity: Signal,
53}
54
55/// 2026-09-26: Run [`crate::coherence::probe`] (the `coherence::CHECKS` questions
56/// and the model-list check) and map it to the round's coherence count and
57/// identity signal. The string is the probe's concern, if any.
58pub async fn coherence_probe(target: &TargetEndpoint, timeout: Duration) -> (Coherence, String) {
59    let report = coherence::probe(target, timeout).await;
60    let identity = match (&report.transport_error, &report.served_instead) {
61        (Some(e), _) => Signal::Fail(one_line(e)),
62        (None, Some(served)) if served.is_empty() => {
63            Signal::Fail("the endpoint has no model loaded".into())
64        }
65        (None, Some(served)) => Signal::Fail(format!(
66            "serving {} — not {}, which this round loaded",
67            served.join(", "),
68            target.model
69        )),
70        (None, None) => Signal::Pass,
71    };
72    let detail = report.concern(target).unwrap_or_default();
73    (
74        Coherence {
75            passed: report.answers.iter().filter(|a| a.passed).count(),
76            total: coherence::CHECKS.len(),
77            identity,
78        },
79        one_line(detail),
80    )
81}
82
83/// 2026-09-26: Structural codegen: does the reply define `fib` with an indented
84/// body (`code_shape`)? The reply is not executed.
85pub async fn codegen_probe(target: &TargetEndpoint, timeout: Duration, budget: usize) -> Signal {
86    let body = with_user(
87        base_body(&target.model, budget),
88        "Write a Python function `fib(n)` that returns the n-th Fibonacci number. \
89         Reply with the code only, no explanation."
90            .into(),
91    );
92    let outcome = match http::chat_stream(target, &body, timeout).await {
93        Ok(o) => o,
94        Err(e) => return Signal::Fail(one_line(format!("{e:#}"))),
95    };
96    match code_shape(&outcome.text) {
97        Ok(()) => Signal::Pass,
98        Err(why) => Signal::Fail(format!("{why}: {}", one_line(&outcome.text))),
99    }
100}
101
102/// 2026-09-26: The structural test on the reply text.
103pub fn code_shape(text: &str) -> Result<(), String> {
104    let lines: Vec<&str> = text.lines().collect();
105    let Some((start, header)) = lines.iter().enumerate().find(|(_, line)| {
106        line.trim_start()
107            .strip_prefix("def fib")
108            .is_some_and(|rest| rest.trim_start().starts_with('('))
109    }) else {
110        return Err("no `def fib` in the reply".into());
111    };
112    if !header.contains(':') {
113        return Err("`def fib` has no parameter list or colon".into());
114    }
115    // 2026-09-26: The body must be on its own indented line; a reply collapsed
116    // onto one line has none.
117    let indented = lines
118        .iter()
119        .skip(start + 1)
120        .any(|l| !l.trim().is_empty() && (l.starts_with(' ') || l.starts_with('\t')));
121    if !indented {
122        return Err("the function has no indented body — the reply collapsed onto one line".into());
123    }
124    Ok(())
125}
126
127/// 2026-09-26: The `get_weather` tool the tool-call probe offers, in OpenAI shape.
128fn weather_tool() -> Value {
129    json!([{
130        "type": "function",
131        "function": {
132            "name": "get_weather",
133            "description": "Get the current weather for a location",
134            "parameters": {
135                "type": "object",
136                "properties": {"location": {"type": "string", "description": "City name"}},
137                "required": ["location"],
138            },
139        },
140    }])
141}
142
143/// 2026-09-26: Did the model call `get_weather` for Paris? A 4xx refusal of the
144/// request is `NotApplicable` (`is_client_rejection`); any other error, or a
145/// reply without that call, is a `Fail`.
146pub async fn tool_call_probe(target: &TargetEndpoint, timeout: Duration, budget: usize) -> Signal {
147    let mut body = with_user(
148        base_body(&target.model, budget),
149        "What is the weather in Paris?".into(),
150    );
151    body["tools"] = weather_tool();
152    body["tool_choice"] = json!("auto");
153    let outcome = match http::chat_stream(target, &body, timeout).await {
154        Ok(o) => o,
155        Err(e) => {
156            // 2026-09-26: A 4xx refusal is a capability gap; a transport failure
157            // is not excused as one.
158            let msg = one_line(format!("{e:#}"));
159            return if is_client_rejection(&msg) {
160                Signal::NotApplicable(msg)
161            } else {
162                Signal::Fail(msg)
163            };
164        }
165    };
166    score_tool_call(&outcome)
167}
168
169fn score_tool_call(outcome: &http::ChatOutcome) -> Signal {
170    let Some(call) = outcome.tool_calls.iter().find(|c| !c.name.is_empty()) else {
171        return Signal::Fail("no tool call in the reply".into());
172    };
173    if call.name != "get_weather" {
174        return Signal::Fail(format!("called {:?}, not get_weather", call.name));
175    }
176    match serde_json::from_str::<Value>(&call.arguments) {
177        Ok(args) => match args.get("location").and_then(Value::as_str) {
178            Some(location) if location.trim().eq_ignore_ascii_case("Paris") => Signal::Pass,
179            Some(location) => Signal::Fail(format!("called for {location:?}, not Paris")),
180            None => Signal::Fail(format!("arguments carry no location: {}", call.arguments)),
181        },
182        Err(e) => Signal::Fail(format!("arguments are not JSON ({e}): {}", call.arguments)),
183    }
184}
185
186/// 2026-09-26: Did the server answer with a 4xx? Only [`crate::http`]'s non-200
187/// wording (`endpoint returned "<status line>"`) counts, and only a 4xx code in
188/// the status line: a bare substring match would also take a `400s` timeout or
189/// a URL on port 8400.
190fn is_client_rejection(msg: &str) -> bool {
191    let Some(rest) = msg.split_once("endpoint returned \"").map(|(_, r)| r) else {
192        return false;
193    };
194    let status = rest.split('"').next().unwrap_or_default();
195    status.split_whitespace().any(|word| {
196        word.len() == 3 && word.starts_with('4') && word.chars().all(|c| c.is_ascii_digit())
197    })
198}
199
200/// 2026-09-26: Needle-in-a-haystack with a `tokens`-token filler.
201pub async fn long_context_probe(
202    target: &TargetEndpoint,
203    timeout: Duration,
204    tokens: usize,
205    tag: &str,
206) -> Signal {
207    let filler = stats::make_prompt(tokens, stats::PromptMode::Natural, tag);
208    // 2026-09-26: Mid-document, at the char boundary at or before half the
209    // filler: a needle at either end can be found by attending only to the
210    // edges.
211    let split = filler.len() / 2;
212    let cut = (0..=split)
213        .rev()
214        .find(|i| filler.is_char_boundary(*i))
215        .unwrap_or(0);
216    let prompt = format!(
217        "{}\nThe secret code is {NEEDLE}.\n{}\n\nWhat is the secret code? Reply with the code only.",
218        &filler[..cut],
219        &filler[cut..]
220    );
221    let body = with_user(base_body(&target.model, 32), prompt);
222    match http::chat_stream(target, &body, timeout).await {
223        Ok(o) if o.text.contains(NEEDLE) => Signal::Pass,
224        Ok(o) => Signal::Fail(format!("needle not recalled: {}", one_line(&o.text))),
225        Err(e) => Signal::Fail(one_line(format!("{e:#}"))),
226    }
227}
228
229/// 2026-09-26: Decode tok/s from the client's TPOT (`ChatOutcome::tpot_ms`) on a
230/// `budget`-token request. `None` when TPOT is undefined (no delta arrived, or
231/// fewer than two completion tokens); a failed request reports 0 with its
232/// error.
233pub async fn tps_probe(
234    target: &TargetEndpoint,
235    timeout: Duration,
236    budget: usize,
237    tag: &str,
238) -> (Option<f64>, Option<String>) {
239    let prompt = stats::make_prompt(128, stats::PromptMode::Count, tag);
240    let body = with_user(base_body(&target.model, budget), prompt);
241    match http::chat_stream(target, &body, timeout).await {
242        Ok(o) => (o.tpot_ms.filter(|v| *v > 0.0).map(|ms| 1000.0 / ms), None),
243        Err(e) => (Some(0.0), Some(one_line(format!("{e:#}")))),
244    }
245}
246
247#[cfg(test)]
248#[path = "probes_tests.rs"]
249mod tests;
metrale_bench/benchmarks/serve_matrix/probes.rs

metrale_bench/benchmarks/serve_matrix/
probes.rs
