metrale_bench/benchmarks/bfcl/
aggregate.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Combining BFCL shard results into the score the gate judges.
4//!
5//! Owner: bench, BFCL benchmark.
6//! Invariants: `aggregate` depends only on each subset's summed `(hits, n)`,
7//! so every partition of the same rows gives the same result through `union`.
8//!
9//! # Why shard scores cannot simply be averaged
10//!
11//! `score.py` does not compute a flat mean. It aggregates hierarchically, and
12//! the weights are the whole difficulty:
13//!
14//! ```text
15//! non_live      = mean[ mean(simple_python, simple_java, simple_javascript),
16//!                       multiple, parallel, parallel_multiple ]      <- four terms
17//! live          = sample-weighted mean over its subsets
18//! hallucination = mean(irrelevance, live_irrelevance)                <- unweighted
19//! normalized    = mean(the categories present)
20//! overall       = flat mean over every scored sample
21//! ```
22//!
23//! So `simple_javascript` (31 rows in the golden draw) weighs the same as
24//! `simple_python` (248 rows), a third of one term each, and `irrelevance` (24
25//! rows) counts equally with `live_irrelevance` (88). A mean of the shards'
26//! `normalized_single_turn_score` is therefore not the whole-set value, and
27//! `overall_accuracy` survives only a sample-count-weighted mean, to the 2
28//! decimal places each shard's JSON is rounded to.
29//!
30//! `score.py` also builds each category from the subsets present, so a subset
31//! missing from a shard changes that category's divisor. `live_parallel` is 16
32//! rows in the golden draw; a quarter of it is four.
33//!
34//! # What this does instead
35//!
36//! Shards report per-subset `(hits, n)` integer counts (`subset_totals`); every
37//! sample scores exactly 0.0 or 1.0 in `score.py`, so nothing is lost. Summing
38//! them across shards and aggregating once applies the weighting to the union,
39//! never to a partial view.
40//!
41//! The arithmetic below is a second implementation of `score.py`'s. A test in
42//! `provision.rs` runs the real `score.py` on a 12-sample fixture and checks
43//! `aggregate` against its output.
44
45use std::collections::BTreeMap;
46
47/// 2026-09-26: `score.py`'s `non_live` subsets, in its order.
48pub const NON_LIVE: [&str; 6] = [
49    "simple_python",
50    "simple_java",
51    "simple_javascript",
52    "multiple",
53    "parallel",
54    "parallel_multiple",
55];
56/// 2026-09-26: `score.py`'s `live` subsets.
57pub const LIVE: [&str; 4] = [
58    "live_simple",
59    "live_multiple",
60    "live_parallel",
61    "live_parallel_multiple",
62];
63/// 2026-09-26: `score.py`'s `hallucination` subsets.
64pub const HALLUCINATION: [&str; 2] = ["irrelevance", "live_irrelevance"];
65/// 2026-09-26: The three `simple_*` subsets collapse to one term inside
66/// `non_live`.
67pub const SIMPLE_AST: [&str; 3] = ["simple_python", "simple_java", "simple_javascript"];
68
69/// 2026-09-26: Hits and sample count for one subset.
70#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
71pub struct Tally {
72    /// 2026-09-26: Samples scored 1.0.
73    pub hits: u64,
74    /// 2026-09-26: Samples scored at all.
75    pub n: u64,
76}
77
78impl Tally {
79    fn mean(self) -> Option<f64> {
80        (self.n > 0).then(|| self.hits as f64 / self.n as f64)
81    }
82}
83
84/// 2026-09-26: The numbers the gate judges.
85#[derive(Clone, Debug, PartialEq)]
86pub struct Aggregate {
87    /// 2026-09-26: Flat mean over every scored sample, x100, rounded to 2dp.
88    pub overall_accuracy: f64,
89    /// 2026-09-26: Unweighted mean of the categories present, x100, rounded to
90    /// 2dp.
91    pub normalized_single_turn_score: f64,
92    /// 2026-09-26: Per category present, x100, rounded to 2dp.
93    pub category_scores: BTreeMap<String, f64>,
94    /// 2026-09-26: Total samples scored.
95    pub total_samples: u64,
96}
97
98fn round2(x: f64) -> f64 {
99    (x * 100.0).round() / 100.0
100}
101
102fn mean(v: &[f64]) -> f64 {
103    if v.is_empty() {
104        0.0
105    } else {
106        v.iter().sum::<f64>() / v.len() as f64
107    }
108}
109
110/// 2026-09-26: Aggregates per-subset tallies with `score.py`'s weighting. Given
111/// the union of every shard's tallies it scores the whole draw; given one
112/// run's tallies it scores that run, which is how the `provision.rs` test
113/// compares it with `score.py`.
114pub fn aggregate(tallies: &BTreeMap<String, Tally>) -> Aggregate {
115    let subset_mean: BTreeMap<&str, f64> = tallies
116        .iter()
117        .filter_map(|(k, t)| t.mean().map(|m| (k.as_str(), m)))
118        .collect();
119
120    let mut category_scores_raw: BTreeMap<String, f64> = BTreeMap::new();
121
122    // 2026-09-26: hallucination: unweighted mean over the subsets present.
123    let present: Vec<f64> = HALLUCINATION
124        .iter()
125        .filter_map(|s| subset_mean.get(s).copied())
126        .collect();
127    if !present.is_empty() {
128        category_scores_raw.insert("hallucination".into(), mean(&present));
129    }
130
131    // 2026-09-26: live: sample-weighted, i.e. the flat mean over live samples.
132    let live: Vec<(&str, f64)> = LIVE
133        .iter()
134        .filter_map(|s| subset_mean.get(s).map(|m| (*s, *m)))
135        .collect();
136    if !live.is_empty() {
137        let total: u64 = live.iter().map(|(s, _)| tallies[*s].n).sum();
138        if total > 0 {
139            let num: f64 = live
140                .iter()
141                .map(|(s, m)| m * tallies[*s].n as f64)
142                .sum::<f64>();
143            category_scores_raw.insert("live".into(), num / total as f64);
144        }
145    }
146
147    // 2026-09-26: non_live: the simple_* subsets present collapse to one term,
148    // then an unweighted mean over that term plus each other subset present.
149    let non_live_present: Vec<&str> = NON_LIVE
150        .iter()
151        .copied()
152        .filter(|s| subset_mean.contains_key(s))
153        .collect();
154    if !non_live_present.is_empty() {
155        let simple: Vec<f64> = SIMPLE_AST
156            .iter()
157            .filter_map(|s| subset_mean.get(s).copied())
158            .collect();
159        let mut top: Vec<f64> = Vec::new();
160        if !simple.is_empty() {
161            top.push(mean(&simple));
162        }
163        for s in &non_live_present {
164            if !SIMPLE_AST.contains(s) {
165                top.push(subset_mean[s]);
166            }
167        }
168        if !top.is_empty() {
169            category_scores_raw.insert("non_live".into(), mean(&top));
170        }
171    }
172
173    let normalized = if category_scores_raw.is_empty() {
174        0.0
175    } else {
176        mean(&category_scores_raw.values().copied().collect::<Vec<_>>())
177    };
178
179    let hits: u64 = tallies.values().map(|t| t.hits).sum();
180    let n: u64 = tallies.values().map(|t| t.n).sum();
181    let overall = if n > 0 { hits as f64 / n as f64 } else { 0.0 };
182
183    Aggregate {
184        overall_accuracy: round2(overall * 100.0),
185        normalized_single_turn_score: round2(normalized * 100.0),
186        category_scores: category_scores_raw
187            .into_iter()
188            .map(|(k, v)| (k, round2(v * 100.0)))
189            .collect(),
190        total_samples: n,
191    }
192}
193
194/// 2026-09-26: Sums shard tallies per subset. A missing or duplicated shard is
195/// not detected here.
196pub fn union(shards: &[BTreeMap<String, Tally>]) -> BTreeMap<String, Tally> {
197    let mut out: BTreeMap<String, Tally> = BTreeMap::new();
198    for shard in shards {
199        for (k, t) in shard {
200            let e = out.entry(k.clone()).or_default();
201            e.hits += t.hits;
202            e.n += t.n;
203        }
204    }
205    out
206}
207
208/// 2026-09-26: Recovers per-subset tallies from a gate record's metrics, the
209/// `subset.<name>.hits` and `subset.<name>.n` keys that `report.rs` writes.
210/// `None` when the record has no tallies, or a subset has hits but no `n`. A
211/// caller treats `None` as "cannot be aggregated", never as an empty
212/// contribution, which would score the group over fewer samples than the draw.
213pub fn tallies_from_metrics(metrics: &BTreeMap<String, f64>) -> Option<BTreeMap<String, Tally>> {
214    let mut out: BTreeMap<String, Tally> = BTreeMap::new();
215    for (k, v) in metrics {
216        let Some(rest) = k.strip_prefix("subset.") else {
217            continue;
218        };
219        if let Some(name) = rest.strip_suffix(".hits") {
220            out.entry(name.to_string()).or_default().hits = *v as u64;
221        } else if let Some(name) = rest.strip_suffix(".n") {
222            out.entry(name.to_string()).or_default().n = *v as u64;
223        }
224    }
225    if out.is_empty() {
226        return None;
227    }
228    // 2026-09-26: Hits with no `n` is malformed; refuse rather than divide by
229    // zero.
230    if out.values().any(|t| t.n == 0 && t.hits > 0) {
231        return None;
232    }
233    Some(out)
234}
metrale_bench/benchmarks/bfcl/aggregate.rs

metrale_bench/benchmarks/bfcl/
aggregate.rs
