metrale_bench/benchmarks/mlperf_agentic/
descriptors.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The registered MLPerf Agentic Inference descriptor,
4//! `mlperf-agentic-subset`, the only one in the registry.
5//!
6//! Owner: bench, mlperf_agentic.
7//! Invariants: none beyond the types.
8
9use super::MlperfAgentic;
10use crate::benchmark::BenchmarkDescriptor;
11use crate::hardware::Sensitivity;
12use crate::metadata::PluginMetadata;
13
14const SUBSET_SUMMARY: &str =
15    "MLPerf Agentic Inference replay, inline-scored — UNRUNNABLE: dataset unpublished";
16pub const SUBSET_METADATA: PluginMetadata = PluginMetadata::metrale(SUBSET_SUMMARY);
17
18pub const SUBSET_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
19    id: "mlperf-agentic-subset",
20    name: "MLPerf agentic (subset)",
21    summary: SUBSET_SUMMARY,
22    detail: "Teacher-forced multi-turn replay of the MLPerf Agentic Inference dataset \
23             (mlcommons/endpoints@7935df4): recorded trajectories are replayed single-stream \
24             under the official immutable sampling params (temp 1.0, top_k 20, top_p 0.95, \
25             presence 1.5, max 8192, preserve_thinking) plus a pinned seed, and scored \
26             in-process by a fixture-verified port of the upstream inline scorer — workflow \
27             intent-code match plus coding bash-executable multiset IoU. \
28             ★ CANNOT RUN TODAY: the official dataset is unpublished (\"MLCommons storage, \
29             link TBD\") and this leg refuses proxies, so it fails loudly at provisioning \
30             until the file ships. No baseline exists; the first measured run on main \
31             becomes one. Reports inline accuracy + OSL only — the SWE-bench Verified leg \
32             of the official three-part gate is a separate live-agent workflow, not a leg.",
33    duration_hint: "unrunnable — dataset TBD",
34    expected_secs: 0,
35    updated: "2026-08-13",
36    needs_confirmation: false,
37    intended_for: Some(crate::benchmark::ModelExpectation {
38        families: &["qwen3.6-35b-a3b"],
39        note: "MLCommons specifies Qwen/Qwen3.6-35B-A3B (BF16); Metrale Engine serves the official \
40               FP8 sibling, which is rules-legal quantization but NOT the named checkpoint \
41               — every number needs that caveat until the three-part accuracy gate is \
42               cleared. Kimi K2.6 (1T) does not fit a GB10.",
43    }),
44    // 2026-09-26: No threshold parameter: the BENCH.toml entry has no bound to
45    // fill one from.
46    threshold_params: &[],
47    // 2026-09-26: Correctness: the scores are intent-code match and
48    // bash-executable IoU. `wall_s` and `output_tok_s` are recorded, but no
49    // bound reads them.
50    sensitivity: Sensitivity::Correctness,
51    ctor: || Box::new(MlperfAgentic::new()),
52};
metrale_bench/benchmarks/mlperf_agentic/descriptors.rs

metrale_bench/benchmarks/mlperf_agentic/
descriptors.rs
