metrale_bench/benchmarks/bfcl/
mini_descriptors.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-10-03: The BFCL `mini` draw ([`DrawSpec::mini`](super::draw::DrawSpec::mini), 192
4//! samples) and the per-model gate ids that run it as a model's cheap accuracy check:
5//! `bfcl-subset-mini` itself is a measurement draw, and `bfcl-subset-mini-nemotron-nano` /
6//! `-nemotron-super` are promotion candidates, each with its own default subject in its model's
7//! BENCH.toml (one required subject per gate id per box class).
8//!
9//! Owner: bench, BFCL benchmark.
10//! Invariants: none beyond the types.
11
12use super::{Bfcl, Variant};
13use crate::benchmark::{BenchmarkDescriptor, ModelExpectation};
14use crate::hardware::Sensitivity;
15use crate::metadata::PluginMetadata;
16
17const MINI_SUMMARY: &str = "The mini n=192 draw (golden's mix at a fifth), AST-scored";
18
19const MINI_DETAIL: &str = "Berkeley Function Calling Leaderboard v4, single-turn, on the mini \
20     draw: the golden draw's three categories at 12.5/2/2 with a floor of 5, which is 192 \
21     samples. A model's cheap accuracy check, run whole (not sharded). Its scores are NOT \
22     comparable to the golden or echolp draws; each gated model carries its own floors. The \
23     record's dataset_fingerprint names the draw: the SHA-256 of the ordered sample ids, N and \
24     the category percentages.";
25
26pub const MINI_METADATA: PluginMetadata = PluginMetadata::metrale(MINI_SUMMARY);
27
28const GATE_THRESHOLD_PARAMS: &[(&str, &str)] = &[
29    ("min_overall", "overall_accuracy"),
30    ("min_normalized", "normalized_single_turn_score"),
31];
32
33pub const SUBSET_MINI_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
34    id: "bfcl-subset-mini",
35    name: "BFCL (subset, mini draw)",
36    summary: MINI_SUMMARY,
37    detail: MINI_DETAIL,
38    duration_hint: "~25 min on a 30B-A3B",
39    expected_secs: 1500,
40    updated: "2026-10-03",
41    needs_confirmation: false,
42    intended_for: None,
43    threshold_params: GATE_THRESHOLD_PARAMS,
44    sensitivity: Sensitivity::Correctness,
45    ctor: || Box::new(Bfcl::new(Variant::SubsetMini)),
46};
47
48/// 2026-10-03: The mini draw on NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4, a separate gate id so the
49/// model has its own required subject (`gate::check::record_is_required_subject`).
50pub const MINI_NEMOTRON_NANO_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
51    id: "bfcl-subset-mini-nemotron-nano",
52    name: "BFCL (mini draw, Nemotron-3-Nano)",
53    summary: MINI_SUMMARY,
54    detail: MINI_DETAIL,
55    duration_hint: "~25 min",
56    expected_secs: 1500,
57    updated: "2026-10-03",
58    needs_confirmation: false,
59    intended_for: Some(ModelExpectation {
60        families: &["nemotron-3-nano-30b-a3b"],
61        note: "The Nemotron-3-Nano accuracy check; its floors are cut from its own runs.",
62    }),
63    threshold_params: GATE_THRESHOLD_PARAMS,
64    sensitivity: Sensitivity::Correctness,
65    ctor: || Box::new(Bfcl::new(Variant::SubsetMini)),
66};
67
68/// 2026-10-03: The same on NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4.
69pub const MINI_NEMOTRON_SUPER_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
70    id: "bfcl-subset-mini-nemotron-super",
71    name: "BFCL (mini draw, Nemotron-3-Super)",
72    summary: MINI_SUMMARY,
73    detail: MINI_DETAIL,
74    duration_hint: "~45 min",
75    expected_secs: 2700,
76    updated: "2026-10-03",
77    needs_confirmation: false,
78    intended_for: Some(ModelExpectation {
79        families: &["nemotron-super-120b-a12b"],
80        note: "The Nemotron-3-Super accuracy check; its floors are cut from its own runs.",
81    }),
82    threshold_params: GATE_THRESHOLD_PARAMS,
83    sensitivity: Sensitivity::Correctness,
84    ctor: || Box::new(Bfcl::new(Variant::SubsetMini)),
85};
metrale_bench/benchmarks/bfcl/mini_descriptors.rs

metrale_bench/benchmarks/bfcl/
mini_descriptors.rs
