metrale_bench/benchmarks/mlperf_agentic/
provision.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Locate and verify the trajectory file, `dataset.jsonl` in the
4//! artifact store's `mlperf-agentic` directory. Nothing downloads it: an
5//! operator places it there.
6//!
7//! Owner: bench, mlperf_agentic.
8//! Invariants: `ensure` returns `Ok` only when the file exists and, if a pin
9//! is set, its full-file SHA256 matches the pin.
10
11use std::io::Read;
12use std::path::{Path, PathBuf};
13
14use anyhow::{Context, Result, bail};
15
16use crate::artifacts::ArtifactStore;
17use crate::plugin::PluginHandle;
18
19pub const PLUGIN_ID: &str = "mlperf-agentic";
20
21/// 2026-09-26: The dataset status the absent-dataset error quotes.
22pub const UPSTREAM_DATASET_STATUS: &str = "mlcommons/endpoints@7935df4 examples/10_Agentic_Inference/README.md: \"The official \
23     MLPerf dataset can be downloaded from MLCommons storage (link TBD)\"";
24
25#[derive(Clone, Debug)]
26pub struct Artifacts {
27    pub dir: PathBuf,
28    pub dataset: PathBuf,
29    /// 2026-09-26: Full-file SHA256 of `dataset`, hex. Written to
30    /// `dataset_summary.json` and into the run's dataset fingerprint.
31    pub file_sha256: String,
32}
33
34/// 2026-09-26: Locate and verify the dataset. `expected_sha256` is the pin:
35/// empty means record only, non-empty means the file must hash to exactly
36/// that (compared case-insensitively).
37pub fn ensure(
38    store: &ArtifactStore,
39    handle: &PluginHandle,
40    expected_sha256: &str,
41) -> Result<Artifacts> {
42    let dir = store.plugin_dir(PLUGIN_ID)?;
43    let dataset = dir.join("dataset.jsonl");
44    if !dataset.is_file() {
45        // 2026-09-26: No proxy or reconstructed dataset: a score from a
46        // different draw is comparable to nothing.
47        bail!(
48            "the MLPerf Agentic Inference dataset is not provisioned: {} does not exist.\n\
49             The official dataset (613 trajectories: 500 Workato workflow + 113 DeepSWE \
50             coding, 30,335 client turns) is NOT yet published — {}. There is no download \
51             URL, license, or auth story to automate, and this leg deliberately refuses to \
52             substitute a proxy or reconstructed dataset: a score from a different draw is \
53             comparable to nothing, however official it looks.\n\
54             When MLCommons publishes the file: place it at the path above, record its \
55             SHA256 as this leg's expected_sha256 parameter (and in BENCH.toml), and run \
56             the calibration protocol in the BENCH.toml note before trusting any number.",
57            dataset.display(),
58            UPSTREAM_DATASET_STATUS,
59        );
60    }
61
62    let (file_sha256, bytes) = sha256_file(&dataset)?;
63    if !expected_sha256.is_empty() && !expected_sha256.eq_ignore_ascii_case(&file_sha256) {
64        bail!(
65            "{} does not match its pin: sha256 {file_sha256} ({bytes} bytes), expected \
66             {expected_sha256}. A replay of the wrong file scores against the wrong ground \
67             truth; refusing to run.",
68            dataset.display()
69        );
70    }
71    handle.info(format!(
72        "dataset {} — {bytes} bytes, sha256 {file_sha256}{}",
73        dataset.display(),
74        if expected_sha256.is_empty() {
75            " (UNPINNED: no expected_sha256 — record-only until the official file ships)"
76        } else {
77            " (pin verified)"
78        }
79    ));
80    std::fs::write(
81        dir.join("dataset_summary.json"),
82        serde_json::to_string_pretty(&serde_json::json!({
83            "file_sha256": file_sha256,
84            "bytes": bytes,
85        }))? + "\n",
86    )?;
87    Ok(Artifacts {
88        dir,
89        dataset,
90        file_sha256,
91    })
92}
93
94/// 2026-09-26: Hex SHA256 and byte count of a file, read in 64 KiB blocks.
95fn sha256_file(path: &Path) -> Result<(String, u64)> {
96    use sha2::{Digest, Sha256};
97    let mut file =
98        std::fs::File::open(path).with_context(|| format!("opening {}", path.display()))?;
99    let mut digest = Sha256::new();
100    let mut buf = [0u8; 64 * 1024];
101    let mut total = 0u64;
102    loop {
103        let n = file
104            .read(&mut buf)
105            .with_context(|| format!("reading {}", path.display()))?;
106        if n == 0 {
107            break;
108        }
109        total += n as u64;
110        digest.update(&buf[..n]);
111    }
112    Ok((metrale_closure::hex_lower(&digest.finalize()), total))
113}
114
115#[cfg(test)]
116mod tests {
117    use super::*;
118
119    fn tmp(name: &str) -> PathBuf {
120        let d =
121            std::env::temp_dir().join(format!("metrale-mlperf-prov-{name}-{}", std::process::id()));
122        let _ = std::fs::remove_dir_all(&d);
123        std::fs::create_dir_all(&d).unwrap();
124        d
125    }
126
127    #[test]
128    fn sha256_matches_a_known_vector() {
129        let dir = tmp("sha");
130        let p = dir.join("f");
131        std::fs::write(&p, b"abc").unwrap();
132        let (sha, n) = sha256_file(&p).unwrap();
133        assert_eq!(n, 3);
134        assert_eq!(
135            sha,
136            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
137        );
138    }
139
140    // 2026-09-26: The absent-dataset failure is tested in `mlperf_tests.rs`,
141    // where a PluginHandle exists to call ensure() with.
142}
metrale_bench/benchmarks/mlperf_agentic/provision.rs

metrale_bench/benchmarks/mlperf_agentic/
provision.rs
