metrale_bench/benchmarks/mlperf_agentic/
scoring.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Rust port of the comparison rules of the MLPerf Agentic
4//! Inference inline scorer, `AgenticInferenceInlineScorer` in
5//! mlcommons/endpoints@7935df4, `src/inference_endpoint/evaluation/scoring.py`:
6//!
7//! * Workflow (`^sim_\d+$` conversation ids): 1.0 if the intent code
8//!   extracted from the model's text is in the ground truth's `intent_codes`,
9//!   else 0.0.
10//! * Coding (every other id): multiset IoU over normalized bash executables
11//!   from `bash` tool calls.
12//!
13//! `scoring_tests.rs` checks every function against
14//! `assets/mlperf-agentic/parity_fixtures.json`, whose expected values
15//! `gen_parity_fixtures.py` produced by executing the upstream class,
16//! including one case per alias-table entry and per shell wrapper. Cases a
17//! port could plausibly get wrong are among them: `"reintent: I042"` still
18//! scores through the bare-token fallback, and executables missing from the
19//! alias table vanish from both sides of the IoU.
20//!
21//! One known divergence: Python's `\s` also matches `\x1c`-`\x1f`, and
22//! [`char::is_whitespace`] does not. It affects only control characters
23//! between `intent:` and the code.
24//!
25//! Owner: bench, mlperf_agentic.
26//! Invariants: for a `gt_turn` that `has_ground_truth` accepts, `score_turn`
27//! is in 0.0..=1.0.
28
29use std::collections::BTreeMap;
30
31use serde_json::Value;
32
33/// 2026-09-26: Which scoring rule applies, decided by conversation id:
34/// `^sim_\d+$` is workflow, anything else is coding.
35#[derive(Clone, Copy, Debug, PartialEq, Eq)]
36pub enum Domain {
37    Workflow,
38    Coding,
39}
40
41pub fn domain_of(conversation_id: &str) -> Domain {
42    match conversation_id.strip_prefix("sim_") {
43        Some(rest) if !rest.is_empty() && rest.bytes().all(|b| b.is_ascii_digit()) => {
44            Domain::Workflow
45        }
46        _ => Domain::Coding,
47    }
48}
49
50/// 2026-09-26: Upstream `_EXECUTABLE_ALIASES`; the test
51/// `the_alias_table_has_exactly_the_upstream_keys` checks the keys against the
52/// fixtures. Executables not in this table are dropped from the multiset, on
53/// both sides of the IoU.
54const EXECUTABLE_ALIASES: [(&str, &str); 58] = [
55    ("python", "python"),
56    ("python2", "python"),
57    ("python3", "python"),
58    ("py", "python"),
59    ("pip", "pip"),
60    ("pip3", "pip"),
61    ("pytest", "pytest"),
62    ("pylint", "pylint"),
63    ("sphinx-build", "sphinx"),
64    ("sphinx-quickstart", "sphinx"),
65    ("cython", "cython"),
66    ("make", "make"),
67    ("conda", "conda"),
68    ("cat", "cat"),
69    ("head", "head"),
70    ("tail", "tail"),
71    ("less", "cat"),
72    ("more", "cat"),
73    ("wc", "wc"),
74    ("diff", "diff"),
75    ("grep", "grep"),
76    ("egrep", "grep"),
77    ("fgrep", "grep"),
78    ("rg", "grep"),
79    ("ag", "grep"),
80    ("sed", "sed"),
81    ("awk", "awk"),
82    ("gawk", "awk"),
83    ("tr", "tr"),
84    ("sort", "sort"),
85    ("uniq", "uniq"),
86    ("cut", "cut"),
87    ("find", "find"),
88    ("ls", "ls"),
89    ("locate", "find"),
90    ("xargs", "xargs"),
91    ("cp", "cp"),
92    ("mv", "mv"),
93    ("rm", "rm"),
94    ("mkdir", "mkdir"),
95    ("touch", "touch"),
96    ("tee", "tee"),
97    ("source", "source"),
98    (".", "source"),
99    ("which", "which"),
100    ("alias", "alias"),
101    ("unset", "unset"),
102    ("export", "export"),
103    ("git", "git"),
104    ("curl", "curl"),
105    ("wget", "curl"),
106    ("true", "true"),
107    ("false", "false"),
108    ("timeout", "timeout"),
109    ("date", "date"),
110    ("apt-get", "apt"),
111    ("apt", "apt"),
112    ("yum", "yum"),
113];
114
115/// 2026-09-26: Upstream `_SHELL_WRAPPERS`: tokens skipped at the front of a
116/// stage before the executable is read.
117const SHELL_WRAPPERS: [&str; 6] = ["env", "time", "nice", "sudo", "exec", "command"];
118
119#[cfg(test)]
120pub(super) fn alias_table() -> &'static [(&'static str, &'static str)] {
121    &EXECUTABLE_ALIASES
122}
123
124/// 2026-09-26: Ground-truth intent codes from an expected assistant turn:
125/// strings from `intent_codes`, upper-cased, sorted and deduplicated, empty
126/// and non-string entries dropped. An empty set means the turn has no workflow
127/// ground truth and is excluded from the denominator.
128pub fn ground_truth_intents(turn: &Value) -> Vec<String> {
129    let Some(codes) = turn.get("intent_codes").and_then(Value::as_array) else {
130        return Vec::new();
131    };
132    let mut out: Vec<String> = codes
133        .iter()
134        .filter_map(Value::as_str)
135        .filter(|s| !s.is_empty())
136        .map(str::to_uppercase)
137        .collect();
138    out.sort();
139    out.dedup();
140    out
141}
142
143/// 2026-09-26: The model's intent code: the explicit `intent: I123` form,
144/// searched over `reasoning_content` then `content` (the explicit pass covers
145/// both fields before the fallback runs over either), then the last bare
146/// `I123` token as fallback. The explicit form is case-insensitive and
147/// upper-cased; the bare fallback is case-sensitive, as the fixture
148/// `"maybe i123"` pins.
149pub fn model_intent(turn: &Value) -> Option<String> {
150    let fields = ["reasoning_content", "content"];
151    for field in fields {
152        if let Some(text) = turn.get(field).and_then(Value::as_str)
153            && let Some(code) = explicit_intent(text)
154        {
155            return Some(code);
156        }
157    }
158    for field in fields {
159        if let Some(text) = turn.get(field).and_then(Value::as_str)
160            && let Some(code) = last_bare_intent(text)
161        {
162            return Some(code);
163        }
164    }
165    None
166}
167
168/// 2026-09-26: Python's `\w` for the `\b` boundaries: alphanumerics plus
169/// underscore.
170fn is_word(c: char) -> bool {
171    c.is_alphanumeric() || c == '_'
172}
173
174/// 2026-09-26: `\bintent:\s*(I\d{3})\b`, IGNORECASE, first match.
175fn explicit_intent(text: &str) -> Option<String> {
176    let chars: Vec<char> = text.chars().collect();
177    let n = chars.len();
178    for start in 0..n {
179        if start > 0 && is_word(chars[start - 1]) {
180            continue;
181        }
182        let keyword = "intent:";
183        if start + keyword.len() > n
184            || !chars[start..start + keyword.len()]
185                .iter()
186                .zip(keyword.chars())
187                .all(|(&c, k)| c.to_ascii_lowercase() == k)
188        {
189            continue;
190        }
191        let mut i = start + keyword.len();
192        while i < n && chars[i].is_whitespace() {
193            i += 1;
194        }
195        if let Some(code) = intent_code_at(&chars, i) {
196            return Some(code);
197        }
198    }
199    None
200}
201
202/// 2026-09-26: `\bI(\d{3})\b` (case-sensitive), last match.
203fn last_bare_intent(text: &str) -> Option<String> {
204    let chars: Vec<char> = text.chars().collect();
205    let mut last = None;
206    for start in 0..chars.len() {
207        if chars[start] != 'I' || (start > 0 && is_word(chars[start - 1])) {
208            continue;
209        }
210        if let Some(code) = intent_code_at(&chars, start) {
211            last = Some(code);
212        }
213    }
214    last
215}
216
217/// 2026-09-26: `[Ii]\d{3}` at `i` with a word boundary after the third digit,
218/// upper-cased.
219fn intent_code_at(chars: &[char], i: usize) -> Option<String> {
220    if i + 4 > chars.len()
221        || !matches!(chars[i], 'I' | 'i')
222        || !chars[i + 1..i + 4].iter().all(|c| c.is_ascii_digit())
223        || chars.get(i + 4).copied().is_some_and(is_word)
224    {
225        return None;
226    }
227    Some(format!("I{}{}{}", chars[i + 1], chars[i + 2], chars[i + 3]))
228}
229
230/// 2026-09-26: Normalized bash executables from a turn's tool calls
231/// (upstream `_bash_actions`): only `function.name == "bash"` calls count;
232/// arguments may be an object or a JSON string (malformed JSON skips that
233/// call); `command` is preferred over `cmd` with Python truthiness (an empty
234/// `command` falls through). Per command: quoted spans stripped, split on
235/// `||` / `|` / `&&` / `;` / newline, leading env assignments and shell
236/// wrappers dropped, executable basenamed, lowercased,
237/// version-suffix-stripped and mapped through the alias table.
238pub fn bash_actions(turn: &Value) -> Vec<String> {
239    let Some(calls) = turn.get("tool_calls").and_then(Value::as_array) else {
240        return Vec::new();
241    };
242    let mut actions = Vec::new();
243    for call in calls {
244        let Some(function) = call.get("function").filter(|f| f.is_object()) else {
245            continue;
246        };
247        if function.get("name").and_then(Value::as_str) != Some("bash") {
248            continue;
249        }
250        let arguments = match function.get("arguments") {
251            Some(Value::String(raw)) => match serde_json::from_str::<Value>(raw) {
252                Ok(parsed) => parsed,
253                Err(_) => continue,
254            },
255            Some(v) => v.clone(),
256            None => continue,
257        };
258        if !arguments.is_object() {
259            continue;
260        }
261        // 2026-09-26: `args.get("command") or args.get("cmd")`: Python
262        // truthiness, so an empty or null `command` falls through to `cmd`.
263        let command = [arguments.get("command"), arguments.get("cmd")]
264            .into_iter()
265            .flatten()
266            .find(|v| py_truthy(v));
267        let Some(command) = command.and_then(Value::as_str) else {
268            continue;
269        };
270        collect_stage_actions(command, &mut actions);
271    }
272    actions
273}
274
275/// 2026-09-26: Python truthiness for a JSON value, as `or` sees it.
276fn py_truthy(v: &Value) -> bool {
277    match v {
278        Value::Null => false,
279        Value::Bool(b) => *b,
280        Value::Number(n) => n.as_f64() != Some(0.0),
281        Value::String(s) => !s.is_empty(),
282        Value::Array(a) => !a.is_empty(),
283        Value::Object(o) => !o.is_empty(),
284    }
285}
286
287fn collect_stage_actions(command: &str, actions: &mut Vec<String>) {
288    let unquoted = strip_quoted_spans(command);
289    for stage in split_separators(&unquoted) {
290        let mut tokens = stage.split_whitespace();
291        let executable = loop {
292            match tokens.next() {
293                None => break None,
294                Some(t) if is_env_assignment(t) || SHELL_WRAPPERS.contains(&t) => continue,
295                Some(t) => break Some(t),
296            }
297        };
298        let Some(executable) = executable else {
299            continue;
300        };
301        let base = executable
302            .rsplit('/')
303            .next()
304            .unwrap_or(executable)
305            .to_lowercase();
306        let stripped = strip_version_suffix(&base);
307        if let Some((_, alias)) = EXECUTABLE_ALIASES.iter().find(|(k, _)| *k == stripped) {
308            actions.push((*alias).to_string());
309        }
310    }
311}
312
313/// 2026-09-26: `'…'`, `"…"` (with backslash escapes) and `` `…` `` spans each
314/// replaced by one space. An unterminated quote matches nothing and the quote
315/// character stays in the text.
316fn strip_quoted_spans(command: &str) -> String {
317    let chars: Vec<char> = command.chars().collect();
318    let mut out = String::with_capacity(command.len());
319    let mut i = 0;
320    while i < chars.len() {
321        let c = chars[i];
322        let close = match c {
323            '\'' | '`' => chars[i + 1..]
324                .iter()
325                .position(|&x| x == c)
326                .map(|p| i + 1 + p),
327            '"' => {
328                let mut j = i + 1;
329                loop {
330                    match chars.get(j) {
331                        None => break None,
332                        Some('\\') => j += 2,
333                        Some('"') => break Some(j),
334                        Some(_) => j += 1,
335                    }
336                }
337            }
338            _ => None,
339        };
340        match close {
341            Some(j) => {
342                out.push(' ');
343                i = j + 1;
344            }
345            None => {
346                out.push(c);
347                i += 1;
348            }
349        }
350    }
351    out
352}
353
354/// 2026-09-26: Split on `||`, `|`, `&&`, `;` and newline. A lone `&` is not a
355/// separator.
356fn split_separators(text: &str) -> Vec<String> {
357    let mut stages = Vec::new();
358    let mut current = String::new();
359    let chars: Vec<char> = text.chars().collect();
360    let mut i = 0;
361    while i < chars.len() {
362        let two = (chars[i], chars.get(i + 1).copied());
363        match two {
364            ('|', Some('|')) | ('&', Some('&')) => {
365                stages.push(std::mem::take(&mut current));
366                i += 2;
367            }
368            ('|' | ';' | '\n', _) => {
369                stages.push(std::mem::take(&mut current));
370                i += 1;
371            }
372            (c, _) => {
373                current.push(c);
374                i += 1;
375            }
376        }
377    }
378    stages.push(current);
379    stages
380}
381
382/// 2026-09-26: `^[A-Za-z_][A-Za-z0-9_]*=`.
383fn is_env_assignment(token: &str) -> bool {
384    let bytes = token.as_bytes();
385    if bytes.is_empty() || !(bytes[0].is_ascii_alphabetic() || bytes[0] == b'_') {
386        return false;
387    }
388    bytes[1..]
389        .iter()
390        .position(|&b| !(b.is_ascii_alphanumeric() || b == b'_'))
391        .is_some_and(|p| bytes[1 + p] == b'=')
392}
393
394/// 2026-09-26: Remove `\.\d+(\.\d+)?$`, leftmost match: at most the last two
395/// trailing `.N` groups come off ("python3.11" → "python3", "python3.1.2.3" →
396/// "python3.1").
397fn strip_version_suffix(name: &str) -> &str {
398    let bytes = name.as_bytes();
399    'outer: for start in 0..bytes.len() {
400        if bytes[start] != b'.' {
401            continue;
402        }
403        let mut i = start + 1;
404        for groups_left in [true, false] {
405            let digits_start = i;
406            while i < bytes.len() && bytes[i].is_ascii_digit() {
407                i += 1;
408            }
409            if i == digits_start {
410                continue 'outer;
411            }
412            if i == bytes.len() {
413                return &name[..start];
414            }
415            if !groups_left || bytes[i] != b'.' {
416                continue 'outer;
417            }
418            i += 1;
419        }
420    }
421    name
422}
423
424/// 2026-09-26: One turn's score. `gt_turn` must be scorable (see
425/// [`has_ground_truth`]); a model turn that produced nothing scores 0 and
426/// stays in the denominator.
427pub fn score_turn(domain: Domain, gt_turn: &Value, model_turn: &Value) -> f64 {
428    match domain {
429        Domain::Workflow => {
430            let gt = ground_truth_intents(gt_turn);
431            match model_intent(model_turn) {
432                Some(code) if gt.contains(&code) => 1.0,
433                _ => 0.0,
434            }
435        }
436        Domain::Coding => multiset_iou(&bash_actions(gt_turn), &bash_actions(model_turn)),
437    }
438}
439
440/// 2026-09-26: Whether an expected assistant turn carries scorable ground
441/// truth. Turns without it are excluded from the denominator, unlike issued
442/// but unanswered turns, which stay at 0.
443pub fn has_ground_truth(domain: Domain, gt_turn: &Value) -> bool {
444    match domain {
445        Domain::Workflow => !ground_truth_intents(gt_turn).is_empty(),
446        Domain::Coding => !bash_actions(gt_turn).is_empty(),
447    }
448}
449
450/// 2026-09-26: `sum(gt ∩ model) / sum(gt ∪ model)` over action counts. The
451/// caller must pass a non-empty `gt`, so the union is not zero.
452pub fn multiset_iou(gt: &[String], model: &[String]) -> f64 {
453    let mut counts: BTreeMap<&str, (usize, usize)> = BTreeMap::new();
454    for a in gt {
455        counts.entry(a).or_default().0 += 1;
456    }
457    for a in model {
458        counts.entry(a).or_default().1 += 1;
459    }
460    let (mut intersection, mut union) = (0usize, 0usize);
461    for (g, m) in counts.values() {
462        intersection += g.min(m);
463        union += g.max(m);
464    }
465    intersection as f64 / union as f64
466}
467
468#[cfg(test)]
469#[path = "scoring_tests.rs"]
470mod tests;
metrale_bench/benchmarks/mlperf_agentic/scoring.rs

metrale_bench/benchmarks/mlperf_agentic/
scoring.rs
