metrale_bench/benchmarks/bfcl/
draw.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The BFCL sample draw. `normalized_single_turn_score` depends on
4//! the category mix, so a score is comparable only with scores on the same
5//! draw.
6//!
7//! Two deterministic rules, with no RNG, produce it:
8//!
9//! 1. Selection by category. Asking for `[non_live, live, hallucination]`
10//!    expands to those categories' subsets; `live_relevance` belongs to none of
11//!    them, so it drops out. That exclusion is the difference between n = 1011
12//!    and the golden n = 995.
13//! 2. Per-subset head(n). `n = total if total <= subset_floor else
14//!    max(1, int(total * pct / 100))`, taking the first `n` rows, concatenated
15//!    in sorted subset order.
16//!
17//! Owner: bench, BFCL benchmark.
18//! Invariants: `shard_take` over `0..count` sums to `take` for every `count`
19//! of at least 1.
20
21use std::collections::BTreeMap;
22
23/// 2026-09-26: Every single-turn subset, in the order `provision.py` writes
24/// them.
25pub const SINGLE_TURN_SUBSETS: [&str; 13] = [
26    "simple_python",
27    "simple_java",
28    "simple_javascript",
29    "multiple",
30    "parallel",
31    "parallel_multiple",
32    "live_simple",
33    "live_multiple",
34    "live_parallel",
35    "live_parallel_multiple",
36    "irrelevance",
37    "live_irrelevance",
38    "live_relevance",
39];
40
41/// 2026-09-26: The three scored categories.
42pub const CATEGORIES: [&str; 3] = ["non_live", "live", "hallucination"];
43
44/// 2026-09-26: Which category a subset belongs to. `live_relevance` maps to
45/// `None`: `score.py` scores it per sample but in no category, and a category
46/// selection leaves it out.
47pub fn category_of(subset: &str) -> Option<&'static str> {
48    match subset {
49        "simple_python" | "simple_java" | "simple_javascript" | "multiple" | "parallel"
50        | "parallel_multiple" => Some("non_live"),
51        "live_simple" | "live_multiple" | "live_parallel" | "live_parallel_multiple" => {
52            Some("live")
53        }
54        "irrelevance" | "live_irrelevance" => Some("hallucination"),
55        _ => None,
56    }
57}
58
59/// 2026-09-26: Sampling configuration.
60#[derive(Clone, Debug, PartialEq)]
61pub struct DrawSpec {
62    /// 2026-09-26: Categories to include. Empty means every single-turn
63    /// subset, including the uncategorised `live_relevance`.
64    pub categories: Vec<String>,
65    /// 2026-09-26: Per-category percentage. A selected subset whose category
66    /// has no entry is taken whole.
67    pub category_pct: BTreeMap<String, f64>,
68    /// 2026-09-26: A subset of at most this many rows is taken in full,
69    /// bypassing the percentage, so `live_parallel` (16 rows) is not cut to one
70    /// to three samples.
71    pub subset_floor: Option<usize>,
72}
73
74impl DrawSpec {
75    fn categories_or_all(categories: &[&str]) -> Vec<String> {
76        categories.iter().map(|s| s.to_string()).collect()
77    }
78
79    /// 2026-09-26: The golden MLPerf-edge draw: the three categories at
80    /// 62/10/10 with a floor of 25. On `reference_subset_totals` this is 995.
81    pub fn golden() -> Self {
82        Self {
83            categories: Self::categories_or_all(&CATEGORIES),
84            category_pct: [
85                ("non_live".to_string(), 62.0),
86                ("live".to_string(), 10.0),
87                ("hallucination".to_string(), 10.0),
88            ]
89            .into_iter()
90            .collect(),
91            subset_floor: Some(25),
92        }
93    }
94
95    /// 2026-09-26: Every sample of the three scored categories, no sampling.
96    /// The same categories as `golden`: taking `live_relevance` too would move
97    /// `overall_accuracy` on a subset no category scores.
98    pub fn full() -> Self {
99        Self {
100            categories: Self::categories_or_all(&CATEGORIES),
101            category_pct: BTreeMap::new(),
102            subset_floor: None,
103        }
104    }
105
106    /// 2026-09-26: The `echolp` draw: the three categories at 46/23/12 with a
107    /// floor of 25. On `reference_subset_totals` this is 1004. It weights `live`
108    /// more than twice as heavily as `golden` (23 % vs 10 %), so a score from one
109    /// draw is not comparable to a threshold from the other; each has its own
110    /// baseline.
111    pub fn echolp() -> Self {
112        Self {
113            categories: Self::categories_or_all(&CATEGORIES),
114            category_pct: [
115                ("non_live".to_string(), 46.0),
116                ("live".to_string(), 23.0),
117                ("hallucination".to_string(), 12.0),
118            ]
119            .into_iter()
120            .collect(),
121            subset_floor: Some(25),
122        }
123    }
124
125    /// 2026-10-03: The `mini` draw for a model's cheap correctness suite: the golden draw's
126    /// three categories at about a fifth of its rates (12.5/2/2) with a floor of 5, so the
127    /// category mix stays close to golden. On `reference_subset_totals` this is
128    /// [`MINI_SAMPLES`]. Not comparable to golden or echolp; it carries its own floors.
129    pub fn mini() -> Self {
130        Self {
131            categories: Self::categories_or_all(&CATEGORIES),
132            category_pct: [
133                ("non_live".to_string(), 12.5),
134                ("live".to_string(), 2.0),
135                ("hallucination".to_string(), 2.0),
136            ]
137            .into_iter()
138            .collect(),
139            subset_floor: Some(5),
140        }
141    }
142
143    /// 2026-09-26: Whether this subset is in the selection.
144    pub fn includes(&self, subset: &str) -> bool {
145        if self.categories.is_empty() {
146            return true;
147        }
148        category_of(subset).is_some_and(|c| self.categories.iter().any(|sel| sel == c))
149    }
150
151    /// 2026-09-26: How many rows to keep from a subset holding `total` rows.
152    pub fn take_count(&self, subset: &str, total: usize) -> usize {
153        if total == 0 || !self.includes(subset) {
154            return 0;
155        }
156        if self.subset_floor.is_some_and(|floor| total <= floor) {
157            return total;
158        }
159        match category_of(subset).and_then(|c| self.category_pct.get(c).copied()) {
160            // 2026-09-26: `as usize` truncates a non-negative f64, like the
161            // `int()` in the rule above; `max(1)` keeps a subset from vanishing.
162            Some(pct) => ((total as f64 * pct / 100.0) as usize).max(1),
163            None => total,
164        }
165    }
166}
167
168/// 2026-09-26: Applies the draw to per-subset totals, returning `(subset,
169/// take)` sorted by subset name and without subsets that take nothing.
170pub fn plan(spec: &DrawSpec, totals: &BTreeMap<String, usize>) -> Vec<(String, usize)> {
171    totals
172        .iter()
173        .map(|(subset, total)| (subset.clone(), spec.take_count(subset, *total)))
174        .filter(|(_, take)| *take > 0)
175        .collect()
176}
177
178/// 2026-10-03: The sample count of [`DrawSpec::mini`] on `reference_subset_totals`.
179pub const MINI_SAMPLES: usize = 192;
180
181/// 2026-10-03: The identity of the draw a run scored, for its record's `dataset_fingerprint`:
182/// `draw-sha256:<hex>;n=<N>;category_sample_pct=<cat>:<pct>,...;subset_floor=<f>`. The hash is
183/// over the run's sample ids in the order they were issued, one per line, so two runs share a
184/// fingerprint only when they scored the same samples in the same order; a BFCL score is not
185/// meaningful without it. A shard's fingerprint covers the shard's own slice.
186pub fn fingerprint<'a>(spec: &DrawSpec, sample_ids: impl IntoIterator<Item = &'a str>) -> String {
187    use sha2::{Digest, Sha256};
188    let mut hasher = Sha256::new();
189    let mut n = 0usize;
190    for id in sample_ids {
191        hasher.update(id.as_bytes());
192        hasher.update(b"\n");
193        n += 1;
194    }
195    let digest: String = hasher
196        .finalize()
197        .iter()
198        .map(|b| format!("{b:02x}"))
199        .collect();
200    let pct = spec
201        .category_pct
202        .iter()
203        .map(|(c, p)| format!("{c}:{p}"))
204        .collect::<Vec<_>>()
205        .join(",");
206    let floor = spec
207        .subset_floor
208        .map_or_else(|| "none".to_string(), |f| f.to_string());
209    format!("draw-sha256:{digest};n={n};category_sample_pct={pct};subset_floor={floor}")
210}
211
212/// 2026-09-26: Total sample count for a plan.
213pub fn total(plan: &[(String, usize)]) -> usize {
214    plan.iter().map(|(_, n)| n).sum()
215}
216
217/// 2026-09-26: The BFCL v4 single-turn row counts per subset, the table the
218/// draw and shard tests and `Bfcl::expected_samples` use. `provision.rs`
219/// saves the counts `provision.py` wrote as `dataset_summary.json`; they are
220/// expected to equal these.
221pub fn reference_subset_totals() -> BTreeMap<String, usize> {
222    [
223        ("irrelevance", 240),
224        ("live_irrelevance", 884),
225        ("live_multiple", 1053),
226        ("live_parallel", 16),
227        ("live_parallel_multiple", 24),
228        ("live_relevance", 16),
229        ("live_simple", 258),
230        ("multiple", 200),
231        ("parallel", 200),
232        ("parallel_multiple", 200),
233        ("simple_java", 100),
234        ("simple_javascript", 50),
235        ("simple_python", 400),
236    ]
237    .into_iter()
238    .map(|(k, v)| (k.to_string(), v))
239    .collect()
240}
241
242/// 2026-09-26: How many of a subset's `take` drawn rows shard `index` of
243/// `count` gets; 0 for an index out of range or a count of 0. Rows are striped,
244/// row `i` going to shard `i % count` (`shard_owns`), not cut into contiguous
245/// blocks, so every shard gets a proportional slice of every subset, and
246/// membership depends on position alone.
247pub fn shard_take(take: usize, index: usize, count: usize) -> usize {
248    if count == 0 || index >= count {
249        return 0;
250    }
251    // 2026-09-26: Rows index..take striding by `count` from `index`:
252    // ceil((take - index) / count).
253    take.saturating_sub(index).div_ceil(count)
254}
255
256/// 2026-09-26: Whether row `row` of a subset (0-based, within the drawn rows)
257/// is in shard `index` of `count`.
258pub fn shard_owns(row: usize, index: usize, count: usize) -> bool {
259    count > 0 && index < count && row % count == index
260}
261
262/// 2026-09-26: The per-subset counts one shard runs, without empty subsets.
263pub fn shard_plan(plan: &[(String, usize)], index: usize, count: usize) -> Vec<(String, usize)> {
264    plan.iter()
265        .map(|(s, take)| (s.clone(), shard_take(*take, index, count)))
266        .filter(|(_, n)| *n > 0)
267        .collect()
268}
269
270#[cfg(test)]
271#[path = "draw_tests.rs"]
272mod tests;
metrale_bench/benchmarks/bfcl/draw.rs

metrale_bench/benchmarks/bfcl/
draw.rs
