1#[derive(Debug, Clone, Copy)]
19pub struct Exclusion {
20 pub prefix: &'static str,
21 pub rationale: &'static str,
22}
23
24#[derive(Debug, Clone, Copy)]
26pub struct GateCoverage {
27 pub id: &'static str,
28 pub excludes: &'static [Exclusion],
29}
30
31pub const PERF_PATHS: [&str; 7] = [
34 "crates",
35 "kernels",
36 "Cargo.toml",
37 "Cargo.lock",
38 "vendor",
39 "jinja-templates",
40 "rust-toolchain.toml",
41];
42
43pub const BOUNDARY_FILES: [&str; 15] = [
51 "crates/bench/src/gate/coverage.rs",
52 "crates/bench/src/gate/required.rs",
55 ".github/pr-taxonomy.json",
61 "crates/bench/src/gate/check.rs",
64 "crates/bench/src/gate/check_paths.rs",
66 "crates/bench/src/gate/check_group.rs",
69 "crates/bench/src/gate/scoring.rs",
71 "crates/bench/src/gate/closure.rs",
74 "crates/bench/src/gate/taxon.rs",
77 "crates/bench/src/gate/bench.rs",
79 "crates/bench/src/gate/agreement.rs",
82 "crates/bench/src/gate/hermetic.rs",
86 "crates/bench/src/gate/amnesty.rs",
89 "crates/bench/src/gate/group.rs",
92 "crates/bench/src/gate/recipe_closure.rs",
95];
96
97#[cfg(test)]
104pub(super) fn gate_machinery_files() -> &'static [&'static str] {
105 GATE_MACHINERY_FILES
106}
107
108#[cfg(test)]
109const GATE_MACHINERY_FILES: &[&str] = &[
110 "crates/bench/src/gate/record.rs",
113 "crates/bench/src/gate/record_path.rs",
114 "crates/bench/src/gate/record_write.rs",
117 "crates/bench/src/gate/record_serve.rs",
120 "crates/bench/src/gate/record_env.rs",
123 "crates/bench/src/gate/record_summary.rs",
124 "crates/bench/src/gate/card.rs",
126 "crates/bench/src/gate/check_fmt.rs",
127 "crates/bench/src/gate/telemetry.rs",
128 "crates/bench/src/gate/telemetry_order.rs",
129 "crates/bench/src/gate/codeowners.rs",
131 "crates/bench/src/gate/pr_taxonomy.rs",
134 "crates/bench/src/gate/signing.rs",
137 "crates/bench/src/gate/fixture_baseline.rs",
139 "crates/bench/src/gate/mod.rs",
140];
141
142const NON_COMPILED_KERNEL_FILES: [&str; 1] = ["BENCH.toml"];
150
151#[derive(Debug, Clone, Copy)]
158pub struct TestOnlyRustModule {
159 pub path: &'static str,
160 pub parent: &'static str,
161 pub name: &'static str,
162 pub declared_path: Option<&'static str>,
163}
164
165pub const TEST_ONLY_RUST_MODULES: &[TestOnlyRustModule] = &[
166 TestOnlyRustModule {
167 path: "crates/model-arch/src/seq_state_reserve_tests.rs",
168 parent: "crates/model-arch/src/seq_state_reserve.rs",
169 name: "tests",
170 declared_path: Some("seq_state_reserve_tests.rs"),
171 },
172 TestOnlyRustModule {
173 path: "crates/model-layers/src/layer/release_contract_tests.rs",
174 parent: "crates/model-layers/src/layer.rs",
175 name: "release_contract_tests",
176 declared_path: Some("layer/release_contract_tests.rs"),
177 },
178 TestOnlyRustModule {
179 path: "crates/server/src/main_modules/serve_phases/preflight/per_sequence_state_tests.rs",
180 parent: "crates/server/src/main_modules/serve_phases/preflight/per_sequence_state.rs",
181 name: "tests",
182 declared_path: Some("per_sequence_state_tests.rs"),
183 },
184 TestOnlyRustModule {
185 path: "crates/config/src/tests.rs",
186 parent: "crates/config/src/lib.rs",
187 name: "tests",
188 declared_path: None,
189 },
190 TestOnlyRustModule {
191 path: "crates/config/src/gguf/tests.rs",
192 parent: "crates/config/src/gguf.rs",
193 name: "tests",
194 declared_path: None,
195 },
196 TestOnlyRustModule {
197 path: "crates/config/src/parsers/lora_tests.rs",
198 parent: "crates/config/src/parsers/lora.rs",
199 name: "tests",
200 declared_path: Some("lora_tests.rs"),
201 },
202 TestOnlyRustModule {
203 path: "crates/bench/src/benchmarks/concurrency_tests.rs",
204 parent: "crates/bench/src/benchmarks/concurrency.rs",
205 name: "concurrency_tests",
206 declared_path: Some("concurrency_tests.rs"),
207 },
208 TestOnlyRustModule {
209 path: "crates/bench/src/benchmarks/concurrency_verdict_tests.rs",
210 parent: "crates/bench/src/benchmarks/concurrency.rs",
211 name: "concurrency_verdict_tests",
212 declared_path: Some("concurrency_verdict_tests.rs"),
213 },
214 TestOnlyRustModule {
215 path: "crates/bench/src/benchmarks/concurrency_moe_tests.rs",
216 parent: "crates/bench/src/benchmarks/concurrency.rs",
217 name: "concurrency_moe_tests",
218 declared_path: Some("concurrency_moe_tests.rs"),
219 },
220 TestOnlyRustModule {
221 path: "crates/bench/src/benchmarks/ttft/long_prompt_tests.rs",
222 parent: "crates/bench/src/benchmarks/ttft/long_prompt.rs",
223 name: "tests",
224 declared_path: Some("long_prompt_tests.rs"),
225 },
226];
227
228fn is_test_only_rust_module(path: &str) -> bool {
229 TEST_ONLY_RUST_MODULES
230 .iter()
231 .any(|entry| path == entry.path)
232}
233
234fn is_non_compiled_kernel_file(path: &str) -> bool {
236 let Some(rest) = path.strip_prefix("kernels/") else {
237 return false;
238 };
239 rest.rsplit('/')
240 .next()
241 .is_some_and(|name| NON_COMPILED_KERNEL_FILES.contains(&name))
242}
243
244const GATE_MACHINERY: Exclusion = Exclusion {
248 prefix: "crates/bench/src/gate",
249 rationale: "gate bookkeeping never runs a model; its correctness is covered by cargo test",
250};
251
252const fn other_driver(prefix: &'static str, mine: &'static str) -> Exclusion {
260 Exclusion {
261 prefix,
262 rationale: mine,
263 }
264}
265
266const fn concurrency_driver(prefix: &'static str, rationale: &'static str) -> Exclusion {
270 other_driver(prefix, rationale)
271}
272
273const TTFT_EXCLUDES: &[Exclusion] = &[
274 GATE_MACHINERY,
275 other_driver(
276 "crates/bench/src/benchmarks/bfcl",
277 "the BFCL driver cannot change what a first-token latency probe measures",
278 ),
279 other_driver(
280 "crates/bench/src/benchmarks/agentic",
281 "the agentic driver cannot change what a first-token latency probe measures",
282 ),
283 other_driver(
284 "crates/bench/src/benchmarks/contamination",
285 "the contamination driver cannot change what a first-token latency probe measures",
286 ),
287 other_driver(
288 "crates/bench/src/benchmarks/ssm_poison",
289 "the SSM poison driver cannot change what a first-token latency probe measures",
290 ),
291 concurrency_driver(
292 "crates/bench/src/benchmarks/concurrency.rs",
293 "the concurrency request planner cannot change what a first-token latency probe measures",
294 ),
295 concurrency_driver(
296 "crates/bench/src/benchmarks/concurrency_verdict.rs",
297 "the concurrency verdict cannot change what a first-token latency probe measures",
298 ),
299 concurrency_driver(
300 "crates/bench/src/benchmarks/concurrency_descriptors.rs",
301 "the concurrency request planner cannot change what a first-token latency probe measures",
302 ),
303 concurrency_driver(
304 "crates/bench/src/benchmarks/concurrency_prompt.rs",
305 "the concurrency request planner cannot change what a first-token latency probe measures",
306 ),
307 concurrency_driver(
308 "crates/bench/src/benchmarks/concurrency_cache.rs",
309 "the concurrency request planner cannot change what a first-token latency probe measures",
310 ),
311 concurrency_driver(
312 "crates/bench/src/benchmarks/concurrency_cell.rs",
313 "the concurrency request planner cannot change what a first-token latency probe measures",
314 ),
315 concurrency_driver(
316 "crates/bench/src/benchmarks/concurrency_report.rs",
317 "the concurrency request planner cannot change what a first-token latency probe measures",
318 ),
319 other_driver(
320 "crates/bench/src/benchmarks/kat_equality",
321 "the equality driver cannot change what a first-token latency probe measures",
322 ),
323 other_driver(
324 "crates/bench/src/benchmarks/scheduler_equivalence",
325 "the equivalence driver cannot change what a first-token latency probe measures",
326 ),
327 other_driver(
328 "crates/bench/src/benchmarks/default_tier_boot",
329 "the boot memory check's driver cannot change what a first-token latency probe measures",
330 ),
331 other_driver(
332 "crates/bench/src/benchmarks/spec_cost",
333 "the speculative step cost driver cannot change what a first-token latency probe measures",
334 ),
335];
336
337const BFCL_EXCLUDES: &[Exclusion] = &[
338 GATE_MACHINERY,
339 other_driver(
340 "crates/bench/src/benchmarks/ttft",
341 "the TTFT driver cannot change a tool-calling accuracy score",
342 ),
343 other_driver(
344 "crates/bench/src/benchmarks/agentic",
345 "the agentic driver cannot change a tool-calling accuracy score",
346 ),
347 other_driver(
348 "crates/bench/src/benchmarks/contamination",
349 "the contamination driver cannot change a tool-calling accuracy score",
350 ),
351 other_driver(
352 "crates/bench/src/benchmarks/ssm_poison",
353 "the SSM poison driver cannot change a tool-calling accuracy score",
354 ),
355 concurrency_driver(
356 "crates/bench/src/benchmarks/concurrency.rs",
357 "the concurrency request planner cannot change a tool-calling accuracy score",
358 ),
359 concurrency_driver(
360 "crates/bench/src/benchmarks/concurrency_verdict.rs",
361 "the concurrency verdict cannot change a tool-calling accuracy score",
362 ),
363 concurrency_driver(
364 "crates/bench/src/benchmarks/concurrency_descriptors.rs",
365 "the concurrency request planner cannot change a tool-calling accuracy score",
366 ),
367 concurrency_driver(
368 "crates/bench/src/benchmarks/concurrency_prompt.rs",
369 "the concurrency request planner cannot change a tool-calling accuracy score",
370 ),
371 concurrency_driver(
372 "crates/bench/src/benchmarks/concurrency_cache.rs",
373 "the concurrency request planner cannot change a tool-calling accuracy score",
374 ),
375 concurrency_driver(
376 "crates/bench/src/benchmarks/concurrency_cell.rs",
377 "the concurrency request planner cannot change a tool-calling accuracy score",
378 ),
379 concurrency_driver(
380 "crates/bench/src/benchmarks/concurrency_report.rs",
381 "the concurrency request planner cannot change a tool-calling accuracy score",
382 ),
383 other_driver(
384 "crates/bench/src/benchmarks/kat_equality",
385 "the equality driver cannot change a tool-calling accuracy score",
386 ),
387 other_driver(
388 "crates/bench/src/benchmarks/scheduler_equivalence",
389 "the equivalence driver cannot change a BFCL score",
390 ),
391 other_driver(
392 "crates/bench/src/benchmarks/default_tier_boot",
393 "the boot memory check's driver cannot change a tool-calling accuracy score",
394 ),
395 other_driver(
396 "crates/bench/src/benchmarks/spec_cost",
397 "the speculative step cost driver cannot change a tool-calling accuracy score",
398 ),
399];
400
401const AGENTIC_EXCLUDES: &[Exclusion] = &[
402 GATE_MACHINERY,
403 other_driver(
404 "crates/bench/src/benchmarks/ttft",
405 "the TTFT driver cannot change whether the agent's webserver task succeeds",
406 ),
407 other_driver(
408 "crates/bench/src/benchmarks/bfcl",
409 "the BFCL driver cannot change whether the agent's webserver task succeeds",
410 ),
411 other_driver(
412 "crates/bench/src/benchmarks/contamination",
413 "the contamination driver cannot change whether the agent's webserver task succeeds",
414 ),
415 other_driver(
416 "crates/bench/src/benchmarks/ssm_poison",
417 "the SSM poison driver cannot change whether the agent's webserver task succeeds",
418 ),
419 concurrency_driver(
420 "crates/bench/src/benchmarks/concurrency.rs",
421 "the concurrency request planner cannot change whether the agent's webserver task succeeds",
422 ),
423 concurrency_driver(
424 "crates/bench/src/benchmarks/concurrency_verdict.rs",
425 "the concurrency verdict cannot change whether the agent's webserver task succeeds",
426 ),
427 concurrency_driver(
428 "crates/bench/src/benchmarks/concurrency_descriptors.rs",
429 "the concurrency request planner cannot change whether the agent's webserver task succeeds",
430 ),
431 concurrency_driver(
432 "crates/bench/src/benchmarks/concurrency_prompt.rs",
433 "the concurrency request planner cannot change whether the agent's webserver task succeeds",
434 ),
435 concurrency_driver(
436 "crates/bench/src/benchmarks/concurrency_cache.rs",
437 "the concurrency request planner cannot change whether the agent's webserver task succeeds",
438 ),
439 concurrency_driver(
440 "crates/bench/src/benchmarks/concurrency_cell.rs",
441 "the concurrency request planner cannot change whether the agent's webserver task succeeds",
442 ),
443 concurrency_driver(
444 "crates/bench/src/benchmarks/concurrency_report.rs",
445 "the concurrency request planner cannot change whether the agent's webserver task succeeds",
446 ),
447 other_driver(
448 "crates/bench/src/benchmarks/kat_equality",
449 "the equality driver cannot change whether the agent's webserver task succeeds",
450 ),
451 other_driver(
452 "crates/bench/src/benchmarks/scheduler_equivalence",
453 "the equivalence driver cannot change what the agentic harness measures",
454 ),
455 other_driver(
456 "crates/bench/src/benchmarks/default_tier_boot",
457 "the boot memory check's driver cannot change whether the agent's webserver task succeeds",
458 ),
459 other_driver(
460 "crates/bench/src/benchmarks/spec_cost",
461 "the speculative step cost driver cannot change whether the agent's webserver task succeeds",
462 ),
463];
464
465const SSM_POISON_EXCLUDES: &[Exclusion] = &[
470 GATE_MACHINERY,
471 other_driver(
472 "crates/bench/src/benchmarks/ttft",
473 "the TTFT driver cannot change whether an identical replay returns identical bytes",
474 ),
475 other_driver(
476 "crates/bench/src/benchmarks/bfcl",
477 "the BFCL driver cannot change whether an identical replay returns identical bytes",
478 ),
479 other_driver(
480 "crates/bench/src/benchmarks/agentic",
481 "the agentic driver cannot change whether an identical replay returns identical bytes",
482 ),
483 other_driver(
484 "crates/bench/src/benchmarks/contamination",
485 "the contamination driver cannot change whether an identical replay returns identical bytes",
486 ),
487 concurrency_driver(
488 "crates/bench/src/benchmarks/concurrency.rs",
489 "the concurrency request planner cannot change whether an identical replay returns identical bytes",
490 ),
491 concurrency_driver(
492 "crates/bench/src/benchmarks/concurrency_verdict.rs",
493 "the concurrency verdict cannot change whether an identical replay returns identical bytes",
494 ),
495 concurrency_driver(
496 "crates/bench/src/benchmarks/concurrency_descriptors.rs",
497 "the concurrency request planner cannot change whether an identical replay returns identical bytes",
498 ),
499 concurrency_driver(
500 "crates/bench/src/benchmarks/concurrency_prompt.rs",
501 "the concurrency request planner cannot change whether an identical replay returns identical bytes",
502 ),
503 concurrency_driver(
504 "crates/bench/src/benchmarks/concurrency_cache.rs",
505 "the concurrency request planner cannot change whether an identical replay returns identical bytes",
506 ),
507 concurrency_driver(
508 "crates/bench/src/benchmarks/concurrency_cell.rs",
509 "the concurrency request planner cannot change whether an identical replay returns identical bytes",
510 ),
511 concurrency_driver(
512 "crates/bench/src/benchmarks/concurrency_report.rs",
513 "the concurrency request planner cannot change whether an identical replay returns identical bytes",
514 ),
515 other_driver(
516 "crates/bench/src/benchmarks/kat_equality",
517 "the equality driver cannot change whether an identical replay returns identical bytes",
518 ),
519 other_driver(
520 "crates/bench/src/benchmarks/scheduler_equivalence",
521 "the equivalence driver cannot change whether a replay poisons SSM state",
522 ),
523 other_driver(
524 "crates/bench/src/benchmarks/default_tier_boot",
525 "the boot memory check's driver cannot change whether an identical replay returns identical bytes",
526 ),
527 other_driver(
528 "crates/bench/src/benchmarks/spec_cost",
529 "the speculative step cost driver cannot change whether an identical replay returns identical bytes",
530 ),
531];
532
533const CONCURRENCY_EXCLUDES: &[Exclusion] = &[
537 GATE_MACHINERY,
538 other_driver(
539 "crates/bench/src/benchmarks/bfcl",
540 "the BFCL driver cannot change the server's latency/throughput curve",
541 ),
542 other_driver(
543 "crates/bench/src/benchmarks/agentic",
544 "the agentic driver cannot change the server's latency/throughput curve",
545 ),
546 other_driver(
547 "crates/bench/src/benchmarks/contamination",
548 "the contamination driver cannot change the server's latency/throughput curve",
549 ),
550 other_driver(
551 "crates/bench/src/benchmarks/ttft",
552 "the TTFT driver issues single requests client-side; it cannot change how fast the \
553 server answers a batch of 32",
554 ),
555 other_driver(
556 "crates/bench/src/benchmarks/kat_equality",
557 "the equality driver cannot change the server's latency/throughput curve",
558 ),
559 other_driver(
560 "crates/bench/src/benchmarks/scheduler_equivalence",
561 "the equivalence driver cannot change a concurrency ladder's numbers",
562 ),
563 other_driver(
564 "crates/bench/src/benchmarks/default_tier_boot",
565 "the boot memory check's driver cannot change the server's latency/throughput curve",
566 ),
567 other_driver(
568 "crates/bench/src/benchmarks/spec_cost",
569 "the speculative step cost driver cannot change the server's latency/throughput curve",
570 ),
571];
572
573const DECODE_FLOOR_EXCLUDES: &[Exclusion] = &[
577 GATE_MACHINERY,
578 other_driver(
579 "crates/bench/src/benchmarks/bfcl",
580 "the BFCL driver cannot change the server's single-user decode rate",
581 ),
582 other_driver(
583 "crates/bench/src/benchmarks/agentic",
584 "the agentic driver cannot change the server's single-user decode rate",
585 ),
586 other_driver(
587 "crates/bench/src/benchmarks/contamination",
588 "the contamination driver cannot change the server's single-user decode rate",
589 ),
590 other_driver(
591 "crates/bench/src/benchmarks/ttft",
592 "the TTFT driver cannot change the server's single-user decode rate",
593 ),
594 other_driver(
595 "crates/bench/src/benchmarks/ssm_poison",
596 "the SSM poison driver cannot change the server's single-user decode rate",
597 ),
598 concurrency_driver(
599 "crates/bench/src/benchmarks/concurrency.rs",
600 "the concurrency request planner cannot change the server's single-user decode rate",
601 ),
602 concurrency_driver(
603 "crates/bench/src/benchmarks/concurrency_verdict.rs",
604 "the concurrency verdict cannot change the server's single-user decode rate",
605 ),
606 concurrency_driver(
607 "crates/bench/src/benchmarks/concurrency_descriptors.rs",
608 "the concurrency request planner cannot change the server's single-user decode rate",
609 ),
610 concurrency_driver(
611 "crates/bench/src/benchmarks/concurrency_prompt.rs",
612 "the concurrency request planner cannot change the server's single-user decode rate",
613 ),
614 concurrency_driver(
615 "crates/bench/src/benchmarks/concurrency_cache.rs",
616 "the concurrency request planner cannot change the server's single-user decode rate",
617 ),
618 concurrency_driver(
619 "crates/bench/src/benchmarks/concurrency_cell.rs",
620 "the concurrency request planner cannot change the server's single-user decode rate",
621 ),
622 concurrency_driver(
623 "crates/bench/src/benchmarks/concurrency_report.rs",
624 "the concurrency request planner cannot change the server's single-user decode rate",
625 ),
626 other_driver(
627 "crates/bench/src/benchmarks/kat_equality",
628 "the equality driver cannot change the server's single-user decode rate",
629 ),
630 other_driver(
631 "crates/bench/src/benchmarks/scheduler_equivalence",
632 "the equivalence driver cannot change the decode floor",
633 ),
634 other_driver(
635 "crates/bench/src/benchmarks/default_tier_boot",
636 "the boot memory check's driver cannot change the server's single-user decode rate",
637 ),
638 other_driver(
639 "crates/bench/src/benchmarks/spec_cost",
640 "the speculative step cost driver cannot change the server's single-user decode rate",
641 ),
642];
643
644const KAT_EQUALITY_EXCLUDES: &[Exclusion] = &[
648 GATE_MACHINERY,
649 other_driver(
650 "crates/bench/src/benchmarks/ttft",
651 "the TTFT driver cannot change whether a reply depends on what ran before it",
652 ),
653 other_driver(
654 "crates/bench/src/benchmarks/agentic",
655 "the agentic driver cannot change whether a reply depends on what ran before it",
656 ),
657 other_driver(
658 "crates/bench/src/benchmarks/ssm_poison",
659 "the SSM poison driver cannot change whether a reply depends on what ran before it",
660 ),
661 other_driver(
662 "crates/bench/src/benchmarks/contamination",
663 "the contamination driver cannot change whether a reply depends on what ran before it",
664 ),
665 concurrency_driver(
666 "crates/bench/src/benchmarks/concurrency.rs",
667 "the concurrency request planner cannot change whether a reply depends on what ran before it",
668 ),
669 concurrency_driver(
670 "crates/bench/src/benchmarks/concurrency_verdict.rs",
671 "the concurrency verdict cannot change whether a reply depends on what ran before it",
672 ),
673 concurrency_driver(
674 "crates/bench/src/benchmarks/concurrency_descriptors.rs",
675 "the concurrency request planner cannot change whether a reply depends on what ran before it",
676 ),
677 concurrency_driver(
678 "crates/bench/src/benchmarks/concurrency_prompt.rs",
679 "the concurrency request planner cannot change whether a reply depends on what ran before it",
680 ),
681 concurrency_driver(
682 "crates/bench/src/benchmarks/concurrency_cache.rs",
683 "the concurrency request planner cannot change whether a reply depends on what ran before it",
684 ),
685 concurrency_driver(
686 "crates/bench/src/benchmarks/concurrency_cell.rs",
687 "the concurrency request planner cannot change whether a reply depends on what ran before it",
688 ),
689 concurrency_driver(
690 "crates/bench/src/benchmarks/concurrency_report.rs",
691 "the concurrency request planner cannot change whether a reply depends on what ran before it",
692 ),
693 other_driver(
694 "crates/bench/src/benchmarks/scheduler_equivalence",
695 "the equivalence driver cannot change whether a reply depends on what ran before it",
696 ),
697 other_driver(
698 "crates/bench/src/benchmarks/default_tier_boot",
699 "the boot memory check's driver cannot change whether a reply depends on what ran before it",
700 ),
701 other_driver(
702 "crates/bench/src/benchmarks/spec_cost",
703 "the speculative step cost driver cannot change whether a reply depends on what ran before it",
704 ),
705];
706
707const SCHEDULER_EQUIVALENCE_EXCLUDES: &[Exclusion] = &[
710 GATE_MACHINERY,
711 other_driver(
712 "crates/bench/src/benchmarks/ttft",
713 "the TTFT driver cannot change whether two routers answer a sample alike",
714 ),
715 other_driver(
716 "crates/bench/src/benchmarks/agentic",
717 "the agentic driver cannot change whether two routers answer a sample alike",
718 ),
719 other_driver(
720 "crates/bench/src/benchmarks/ssm_poison",
721 "the SSM poison driver cannot change whether two routers answer a sample alike",
722 ),
723 other_driver(
724 "crates/bench/src/benchmarks/contamination",
725 "the contamination driver cannot change whether two routers answer a sample alike",
726 ),
727 other_driver(
728 "crates/bench/src/benchmarks/kat_equality",
729 "the order-independence driver cannot change whether two routers answer a sample alike",
730 ),
731 concurrency_driver(
732 "crates/bench/src/benchmarks/concurrency.rs",
733 "the concurrency request planner cannot change whether two routers answer a sample alike",
734 ),
735 concurrency_driver(
736 "crates/bench/src/benchmarks/concurrency_verdict.rs",
737 "the concurrency verdict cannot change whether two routers answer a sample alike",
738 ),
739 concurrency_driver(
740 "crates/bench/src/benchmarks/concurrency_descriptors.rs",
741 "the concurrency request planner cannot change whether two routers answer a sample alike",
742 ),
743 concurrency_driver(
744 "crates/bench/src/benchmarks/concurrency_prompt.rs",
745 "the concurrency request planner cannot change whether two routers answer a sample alike",
746 ),
747 concurrency_driver(
748 "crates/bench/src/benchmarks/concurrency_cache.rs",
749 "the concurrency request planner cannot change whether two routers answer a sample alike",
750 ),
751 concurrency_driver(
752 "crates/bench/src/benchmarks/concurrency_cell.rs",
753 "the concurrency request planner cannot change whether two routers answer a sample alike",
754 ),
755 concurrency_driver(
756 "crates/bench/src/benchmarks/concurrency_report.rs",
757 "the concurrency request planner cannot change whether two routers answer a sample alike",
758 ),
759 other_driver(
760 "crates/bench/src/benchmarks/default_tier_boot",
761 "the boot memory check's driver cannot change whether two routers answer a sample alike",
762 ),
763 other_driver(
764 "crates/bench/src/benchmarks/spec_cost",
765 "the speculative step cost driver cannot change whether two routers answer a sample alike",
766 ),
767];
768
769const CONTAMINATION_EXCLUDES: &[Exclusion] = &[
772 GATE_MACHINERY,
773 other_driver(
774 "crates/bench/src/benchmarks/ttft",
775 "the TTFT driver cannot change whether one request's state leaks into another's output",
776 ),
777 other_driver(
778 "crates/bench/src/benchmarks/bfcl",
779 "the BFCL driver cannot change whether one request's state leaks into another's output",
780 ),
781 other_driver(
782 "crates/bench/src/benchmarks/agentic",
783 "the agentic driver cannot change whether one request's state leaks into another's output",
784 ),
785 concurrency_driver(
786 "crates/bench/src/benchmarks/concurrency.rs",
787 "the concurrency request planner cannot change whether one request's state leaks into another",
788 ),
789 concurrency_driver(
790 "crates/bench/src/benchmarks/concurrency_verdict.rs",
791 "the concurrency verdict cannot change whether one request's state leaks into another",
792 ),
793 concurrency_driver(
794 "crates/bench/src/benchmarks/concurrency_descriptors.rs",
795 "the concurrency request planner cannot change whether one request's state leaks into another",
796 ),
797 concurrency_driver(
798 "crates/bench/src/benchmarks/concurrency_prompt.rs",
799 "the concurrency request planner cannot change whether one request's state leaks into another",
800 ),
801 concurrency_driver(
802 "crates/bench/src/benchmarks/concurrency_cache.rs",
803 "the concurrency request planner cannot change whether one request's state leaks into another",
804 ),
805 concurrency_driver(
806 "crates/bench/src/benchmarks/concurrency_cell.rs",
807 "the concurrency request planner cannot change whether one request's state leaks into another",
808 ),
809 concurrency_driver(
810 "crates/bench/src/benchmarks/concurrency_report.rs",
811 "the concurrency request planner cannot change whether one request's state leaks into another",
812 ),
813 other_driver(
814 "crates/bench/src/benchmarks/kat_equality",
815 "the equality driver cannot change whether one request's state leaks into \
816 another's output",
817 ),
818 other_driver(
819 "crates/bench/src/benchmarks/scheduler_equivalence",
820 "the equivalence driver cannot change whether one request's state leaks into another's output",
821 ),
822 other_driver(
823 "crates/bench/src/benchmarks/default_tier_boot",
824 "the boot memory check's driver cannot change whether one request's state leaks into another",
825 ),
826 other_driver(
827 "crates/bench/src/benchmarks/spec_cost",
828 "the speculative step cost driver cannot change whether one request's state leaks into another",
829 ),
830];
831
832const VISION_EXCLUDES: &[Exclusion] = &[
835 GATE_MACHINERY,
836 other_driver(
837 "crates/bench/src/benchmarks/ttft",
838 "the TTFT driver cannot change how an image is patched or how many tokens it becomes",
839 ),
840 other_driver(
841 "crates/bench/src/benchmarks/bfcl",
842 "the BFCL driver cannot change how an image is patched or how many tokens it becomes",
843 ),
844 other_driver(
845 "crates/bench/src/benchmarks/agentic",
846 "the agentic driver cannot change how an image is patched or how many tokens it becomes",
847 ),
848 other_driver(
849 "crates/bench/src/benchmarks/contamination",
850 "the contamination driver cannot change how an image is patched or how many tokens it becomes",
851 ),
852 concurrency_driver(
853 "crates/bench/src/benchmarks/concurrency.rs",
854 "the concurrency request planner cannot change image preprocessing or encoder tokens",
855 ),
856 concurrency_driver(
857 "crates/bench/src/benchmarks/concurrency_verdict.rs",
858 "the concurrency verdict cannot change image preprocessing or encoder tokens",
859 ),
860 concurrency_driver(
861 "crates/bench/src/benchmarks/concurrency_descriptors.rs",
862 "the concurrency request planner cannot change image preprocessing or encoder tokens",
863 ),
864 concurrency_driver(
865 "crates/bench/src/benchmarks/concurrency_prompt.rs",
866 "the concurrency request planner cannot change image preprocessing or encoder tokens",
867 ),
868 concurrency_driver(
869 "crates/bench/src/benchmarks/concurrency_cache.rs",
870 "the concurrency request planner cannot change image preprocessing or encoder tokens",
871 ),
872 concurrency_driver(
873 "crates/bench/src/benchmarks/concurrency_cell.rs",
874 "the concurrency request planner cannot change image preprocessing or encoder tokens",
875 ),
876 concurrency_driver(
877 "crates/bench/src/benchmarks/concurrency_report.rs",
878 "the concurrency request planner cannot change image preprocessing or encoder tokens",
879 ),
880 other_driver(
881 "crates/bench/src/benchmarks/kat_equality",
882 "the equality driver cannot change how an image is patched or how many tokens \
883 it becomes",
884 ),
885 other_driver(
886 "crates/bench/src/benchmarks/scheduler_equivalence",
887 "the equivalence driver cannot change how an image is patched or how many tokens it becomes",
888 ),
889 other_driver(
890 "crates/bench/src/benchmarks/default_tier_boot",
891 "the boot memory check's driver cannot change how an image is patched or how many tokens it becomes",
892 ),
893 other_driver(
894 "crates/bench/src/benchmarks/spec_cost",
895 "the speculative step cost driver cannot change how an image is patched or how many tokens it becomes",
896 ),
897];
898
899const DEFAULT_TIER_BOOT_EXCLUDES: &[Exclusion] = &[
903 GATE_MACHINERY,
904 other_driver(
905 "crates/bench/src/benchmarks/ttft",
906 "the TTFT driver cannot change how much memory the serve plans or holds at boot",
907 ),
908 other_driver(
909 "crates/bench/src/benchmarks/bfcl",
910 "the BFCL driver cannot change how much memory the serve plans or holds at boot",
911 ),
912 other_driver(
913 "crates/bench/src/benchmarks/agentic",
914 "the agentic driver cannot change how much memory the serve plans or holds at boot",
915 ),
916 other_driver(
917 "crates/bench/src/benchmarks/contamination",
918 "the contamination driver cannot change how much memory the serve plans or holds at boot",
919 ),
920 other_driver(
921 "crates/bench/src/benchmarks/ssm_poison",
922 "the SSM poison driver cannot change how much memory the serve plans or holds at boot",
923 ),
924 concurrency_driver(
925 "crates/bench/src/benchmarks/concurrency.rs",
926 "the concurrency request planner cannot change how much memory the serve plans or holds at boot",
927 ),
928 concurrency_driver(
929 "crates/bench/src/benchmarks/concurrency_verdict.rs",
930 "the concurrency verdict cannot change how much memory the serve plans or holds at boot",
931 ),
932 concurrency_driver(
933 "crates/bench/src/benchmarks/concurrency_descriptors.rs",
934 "the concurrency request planner cannot change how much memory the serve plans or holds at boot",
935 ),
936 concurrency_driver(
937 "crates/bench/src/benchmarks/concurrency_prompt.rs",
938 "the concurrency request planner cannot change how much memory the serve plans or holds at boot",
939 ),
940 concurrency_driver(
941 "crates/bench/src/benchmarks/concurrency_cache.rs",
942 "the concurrency request planner cannot change how much memory the serve plans or holds at boot",
943 ),
944 concurrency_driver(
945 "crates/bench/src/benchmarks/concurrency_cell.rs",
946 "the concurrency request planner cannot change how much memory the serve plans or holds at boot",
947 ),
948 concurrency_driver(
949 "crates/bench/src/benchmarks/concurrency_report.rs",
950 "the concurrency request planner cannot change how much memory the serve plans or holds at boot",
951 ),
952 other_driver(
953 "crates/bench/src/benchmarks/kat_equality",
954 "the equality driver cannot change how much memory the serve plans or holds at boot",
955 ),
956 other_driver(
957 "crates/bench/src/benchmarks/scheduler_equivalence",
958 "the equivalence driver cannot change how much memory the serve plans or holds at boot",
959 ),
960 other_driver(
961 "crates/bench/src/benchmarks/spec_cost",
962 "the speculative step cost driver cannot change how much memory the serve plans or holds at boot",
963 ),
964];
965
966pub const REQUIRED: [GateCoverage; 17] = [
969 GateCoverage {
970 id: "agentic-webserver",
971 excludes: AGENTIC_EXCLUDES,
972 },
973 GateCoverage {
976 id: "vision-fidelity",
977 excludes: VISION_EXCLUDES,
978 },
979 GateCoverage {
980 id: "video-fidelity",
981 excludes: VISION_EXCLUDES,
982 },
983 GateCoverage {
984 id: "ttft-warm-gate",
985 excludes: TTFT_EXCLUDES,
986 },
987 GateCoverage {
988 id: "ttft-cold-gate",
989 excludes: TTFT_EXCLUDES,
990 },
991 GateCoverage {
992 id: "bfcl-subset",
993 excludes: BFCL_EXCLUDES,
994 },
995 GateCoverage {
996 id: "bfcl-subset-echolp",
997 excludes: BFCL_EXCLUDES,
998 },
999 GateCoverage {
1000 id: "ssm-state-poisoning-gate",
1001 excludes: SSM_POISON_EXCLUDES,
1002 },
1003 GateCoverage {
1004 id: "decode-floor",
1005 excludes: DECODE_FLOOR_EXCLUDES,
1006 },
1007 GateCoverage {
1008 id: "concurrency-sweep",
1009 excludes: CONCURRENCY_EXCLUDES,
1010 },
1011 GateCoverage {
1012 id: "concurrency-sweep-dflash2",
1013 excludes: CONCURRENCY_EXCLUDES,
1014 },
1015 GateCoverage {
1016 id: "kat-equality-gate",
1017 excludes: KAT_EQUALITY_EXCLUDES,
1018 },
1019 GateCoverage {
1020 id: "concurrency-sweep-moe",
1021 excludes: CONCURRENCY_EXCLUDES,
1022 },
1023 GateCoverage {
1027 id: "high-isl-ttft-cold",
1028 excludes: TTFT_EXCLUDES,
1029 },
1030 GateCoverage {
1031 id: "high-isl-ttft-warm",
1032 excludes: TTFT_EXCLUDES,
1033 },
1034 GateCoverage {
1035 id: "high-isl-ttft-cold-moe",
1036 excludes: TTFT_EXCLUDES,
1037 },
1038 GateCoverage {
1039 id: "high-isl-ttft-warm-moe",
1040 excludes: TTFT_EXCLUDES,
1041 },
1042];
1043
1044pub const PROMOTION_CANDIDATES: &[GateCoverage] = &[
1051 GateCoverage {
1052 id: "cross-contamination",
1053 excludes: CONTAMINATION_EXCLUDES,
1054 },
1055 GateCoverage {
1058 id: "scheduler-equivalence",
1059 excludes: SCHEDULER_EQUIVALENCE_EXCLUDES,
1060 },
1061 GateCoverage {
1064 id: "default-tier-boot",
1065 excludes: DEFAULT_TIER_BOOT_EXCLUDES,
1066 },
1067 GateCoverage {
1070 id: "kat-equality-gate-nemotron-nano",
1071 excludes: KAT_EQUALITY_EXCLUDES,
1072 },
1073 GateCoverage {
1074 id: "kat-equality-gate-nemotron-super",
1075 excludes: KAT_EQUALITY_EXCLUDES,
1076 },
1077 GateCoverage {
1078 id: "bfcl-subset-mini-nemotron-nano",
1079 excludes: BFCL_EXCLUDES,
1080 },
1081 GateCoverage {
1082 id: "bfcl-subset-mini-nemotron-super",
1083 excludes: BFCL_EXCLUDES,
1084 },
1085];
1086
1087pub const NOT_REQUIRED: [(&str, &str); 13] = [
1090 (
1091 "quick-speed-bench",
1092 "a single-user speed probe with no thresholds and no baseline — a MEASUREMENT tool, \
1093 deliberately never a gate: the required gates already cost hours per PR, and \
1094 its warm-path numbers (primed prefix cache + SSM snapshot) are not regression evidence",
1095 ),
1096 (
1097 "bfcl-full",
1098 "the unsampled ~3600-sample draw; the two subset gates cover the same code at a \
1099 fraction of the GPU time, and a full run would dominate every PR",
1100 ),
1101 (
1102 "serve-matrix",
1103 "a multi-checkpoint survey used for release notes; it measures breadth, not regression",
1104 ),
1105 (
1106 "cross-contamination",
1107 "not required YET: a promotion candidate (see PROMOTION_CANDIDATES) run on release cuts \
1108 and recorded as debt until it has proven itself; a fresh gate that fails on day one \
1109 would train people to override it",
1110 ),
1111 (
1112 "scheduler-equivalence",
1113 "not required YET: a promotion candidate (see PROMOTION_CANDIDATES). It certifies the \
1114 asynchronous device router against the synchronous one, and `--scheduler-config \
1115 async` is not the default; it becomes required when that changes, on a green run \
1116 on every served model",
1117 ),
1118 (
1119 "kat-equality-gate-nemotron-nano",
1120 "not required YET: a promotion candidate (see PROMOTION_CANDIDATES), the Nemotron-3-Nano \
1121 correctness suite. Its BENCH.toml entry is unmeasured; it becomes required once three \
1122 runs set its bounds (measure, then declare)",
1123 ),
1124 (
1125 "kat-equality-gate-nemotron-super",
1126 "not required YET: a promotion candidate (see PROMOTION_CANDIDATES), the Nemotron-3-Super \
1127 correctness suite. Its BENCH.toml entry is unmeasured; it becomes required once three \
1128 runs set its bounds (measure, then declare)",
1129 ),
1130 (
1131 "bfcl-subset-mini-nemotron-nano",
1132 "not required YET: a promotion candidate (see PROMOTION_CANDIDATES), the Nemotron-3-Nano \
1133 accuracy check on the mini draw. Its floors are cut from three measured runs first",
1134 ),
1135 (
1136 "bfcl-subset-mini-nemotron-super",
1137 "not required YET: a promotion candidate (see PROMOTION_CANDIDATES), the Nemotron-3-Super \
1138 accuracy check on the mini draw. Its floors are cut from three measured runs first",
1139 ),
1140 (
1141 "bfcl-subset-mini",
1142 "the 192-sample mini draw itself, a measurement draw with no baseline: models are gated \
1143 on it through their own gate ids (bfcl-subset-mini-<model>), each with its own floors",
1144 ),
1145 (
1146 "default-tier-boot",
1147 "not required YET: a promotion candidate (see PROMOTION_CANDIDATES). It lands \
1148 informational, with no bounds, so its first records measure the default tier's memory \
1149 plan; it becomes required once BENCH.toml declares bounds from them",
1150 ),
1151 (
1152 "spec-cost",
1153 "a calibration MEASUREMENT for the speculative cost model, with no baseline and no \
1154 bounds: it records the per-step verify and draft cost at each batch width so the \
1155 cost table can be fitted, and a calibration has nothing to regress against",
1156 ),
1157 (
1158 "mlperf-agentic-subset",
1159 "not RUNNABLE yet: the official MLPerf Agentic Inference dataset is unpublished \
1160 upstream (mlcommons/endpoints@7935df4: \"MLCommons storage (link TBD)\"), so the leg \
1161 cannot be run, scored, or timed, and it refuses proxy datasets on purpose. Not a \
1162 promotion candidate either — a candidate accrues debt rows, and debt nobody can \
1163 discharge is worse than no row. Promote only after the dataset ships and a \
1164 calibration run sizes a <2 h draw",
1165 ),
1166];
1167
1168fn under(path: &str, entry: &str) -> bool {
1172 path == entry
1173 || path
1174 .strip_prefix(entry)
1175 .is_some_and(|rest| rest.starts_with('/'))
1176}
1177
1178pub fn on_boundary(path: &str) -> bool {
1180 PERF_PATHS.iter().any(|entry| under(path, entry))
1181}
1182
1183pub fn invalidates(gate: &GateCoverage, path: &str) -> bool {
1193 if BOUNDARY_FILES.iter().any(|f| under(path, f)) {
1194 return true;
1195 }
1196 if is_test_only_rust_module(path) {
1197 return false;
1198 }
1199 if is_non_compiled_kernel_file(path) {
1200 return false;
1201 }
1202 if !on_boundary(path) {
1203 return false;
1204 }
1205 !gate.excludes.iter().any(|e| under(path, e.prefix))
1206}
1207
1208pub fn invalidated_by<'a, I>(paths: I) -> Vec<&'static str>
1211where
1212 I: IntoIterator<Item = &'a str>,
1213{
1214 let paths: Vec<&str> = paths.into_iter().collect();
1215 REQUIRED
1216 .iter()
1217 .filter(|gate| paths.iter().any(|p| invalidates(gate, p)))
1218 .map(|gate| gate.id)
1219 .collect()
1220}
1221
1222pub fn find(id: &str) -> Option<&'static GateCoverage> {
1224 REQUIRED.iter().find(|g| g.id == id)
1225}
1226
1227pub fn promotion_debt<'a, I>(paths: I) -> Vec<&'static str>
1230where
1231 I: IntoIterator<Item = &'a str>,
1232{
1233 let paths: Vec<&str> = paths.into_iter().collect();
1234 PROMOTION_CANDIDATES
1235 .iter()
1236 .filter(|gate| paths.iter().any(|p| invalidates(gate, p)))
1237 .map(|gate| gate.id)
1238 .collect()
1239}