metrale_bench/benchmarks/bfcl/
sensitive.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The BFCL samples whose answer depended on what ran before them.
4//! Measured 2026-09-08: the golden n=995 draw run whole and the same draw run
5//! as four shards disagreed on exactly these `sample_id`s.
6//! The mechanism is cross-request SSM snapshot reuse, described in
7//! [`crate::gate::group`]. The driver warns on each of these it runs and
8//! reports the count as the `known_partition_sensitive` metric; membership
9//! does not change how a sample is scored.
10//!
11//! Owner: bench, BFCL benchmark.
12//! Invariants: [`KNOWN_PARTITION_SENSITIVE`] is sorted and free of duplicates.
13
14/// 2026-09-26: Sorted, because [`is_known`] binary-searches it.
15pub const KNOWN_PARTITION_SENSITIVE: &[&str] = &[
16    "live_irrelevance_14-2-2",
17    "live_irrelevance_15-2-3",
18    "live_irrelevance_16-2-4",
19    "live_irrelevance_2-0-2",
20    "live_irrelevance_40-2-28",
21    "live_irrelevance_47-2-35",
22    "live_irrelevance_55-2-43",
23    "live_irrelevance_71-2-59",
24    "live_irrelevance_79-2-67",
25    "live_irrelevance_8-0-8",
26    "live_multiple_57-22-4",
27    "live_parallel_multiple_3-2-1",
28];
29
30pub fn is_known(sample_id: &str) -> bool {
31    KNOWN_PARTITION_SENSITIVE.binary_search(&sample_id).is_ok()
32}
33
34#[cfg(test)]
35mod tests {
36    use super::*;
37
38    #[test]
39    fn the_list_is_sorted_and_deduplicated_so_binary_search_is_valid() {
40        let mut sorted = KNOWN_PARTITION_SENSITIVE.to_vec();
41        sorted.sort_unstable();
42        sorted.dedup();
43        assert_eq!(sorted, KNOWN_PARTITION_SENSITIVE, "keep the list sorted");
44        assert_eq!(KNOWN_PARTITION_SENSITIVE.len(), 12, "#936 records twelve");
45    }
46
47    #[test]
48    fn every_id_names_a_single_turn_subset() {
49        for id in KNOWN_PARTITION_SENSITIVE {
50            let subset = super::super::draw::SINGLE_TURN_SUBSETS
51                .iter()
52                .filter(|s| id.starts_with(&format!("{s}_")))
53                .max_by_key(|s| s.len());
54            assert!(subset.is_some(), "{id} does not belong to a scored subset");
55        }
56    }
57
58    #[test]
59    fn membership_is_exact() {
60        assert!(is_known("live_irrelevance_2-0-2"));
61        assert!(is_known("live_parallel_multiple_3-2-1"));
62        assert!(!is_known("live_irrelevance_2-0-3"));
63        assert!(!is_known("live_irrelevance_2-0"));
64        assert!(!is_known("simple_python_0"));
65    }
66}
metrale_bench/benchmarks/bfcl/sensitive.rs

metrale_bench/benchmarks/bfcl/
sensitive.rs
