metrale_bench/benchmarks/kat_equality/
nemotron.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-10-03: Per-model gate ids for the KAT equality check on the Nemotron-3 models: the same
4//! driver (`driver::KatEquality`), each id with its own default subject in its model's
5//! BENCH.toml, because a gate id has one required subject per box class
6//! (`gate::check::record_is_required_subject`). Promotion candidates until their BENCH entries
7//! are measured.
8//!
9//! Owner: bench, kat_equality.
10//! Invariants: none beyond the types.
11
12use crate::benchmark::{BenchmarkDescriptor, ModelExpectation};
13use crate::hardware::Sensitivity;
14
15use super::driver::{DESCRIPTOR, KatEquality};
16
17const DETAIL: &str = "The KAT equality check (see `kat-equality-gate`) on one Nemotron-3 \
18     checkpoint, at the sample cap its BENCH.toml entry pins: one BFCL draw in two request \
19     orders at temperature 0, every reply byte-identical across the orders, served hermetic.";
20
21pub const NEMOTRON_NANO_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
22    id: "kat-equality-gate-nemotron-nano",
23    name: "KAT Equality Gate (Nemotron-3-Nano)",
24    summary: DESCRIPTOR.summary,
25    detail: DETAIL,
26    duration_hint: "~20 min at sample_cap 64",
27    expected_secs: 1200,
28    updated: "2026-10-03",
29    needs_confirmation: false,
30    intended_for: Some(ModelExpectation {
31        families: &["nemotron-3-nano-30b-a3b"],
32        note: "The Nemotron-3-Nano order-independence check.",
33    }),
34    threshold_params: &[],
35    sensitivity: Sensitivity::Correctness,
36    ctor: || Box::new(KatEquality::default()),
37};
38
39pub const NEMOTRON_SUPER_DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
40    id: "kat-equality-gate-nemotron-super",
41    name: "KAT Equality Gate (Nemotron-3-Super)",
42    summary: DESCRIPTOR.summary,
43    detail: DETAIL,
44    duration_hint: "~35 min at sample_cap 64",
45    expected_secs: 2100,
46    updated: "2026-10-03",
47    needs_confirmation: false,
48    intended_for: Some(ModelExpectation {
49        families: &["nemotron-super-120b-a12b"],
50        note: "The Nemotron-3-Super order-independence check.",
51    }),
52    threshold_params: &[],
53    sensitivity: Sensitivity::Correctness,
54    ctor: || Box::new(KatEquality::default()),
55};
metrale_bench/benchmarks/kat_equality/nemotron.rs

metrale_bench/benchmarks/kat_equality/
nemotron.rs
