metrale_bench/benchmarks/agentic/
descriptors.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The `agentic-webserver` benchmark's descriptor and metadata.
4//!
5//! Owner: bench, agentic.
6//! Invariants: none beyond the types.
7
8use super::AgenticWebserver;
9use crate::benchmark::BenchmarkDescriptor;
10use crate::hardware::Sensitivity;
11use crate::metadata::PluginMetadata;
12
13const SUMMARY: &str = "N agentic runs: build a working Axum server, then verify it";
14pub const METADATA: PluginMetadata = PluginMetadata::metrale(SUMMARY);
15
16/// 2026-09-26: Under `--pull-request-gate` each model's serve recipe and metric
17/// bounds come from its `BENCH.toml` entry for this gate; for the 35B MoE that
18/// is recipe `qwen3.6/qwen3.6-35b-a3b-fp8-bf16head`.
19pub const DESCRIPTOR: BenchmarkDescriptor = BenchmarkDescriptor {
20    id: "agentic-webserver",
21    name: "Agentic Webserver Test",
22    summary: SUMMARY,
23    detail: "Runs the flagship agentic task N times: the model writes a Rust Axum ping/pong \
24             server, tests it, runs it and tears it down, using bash/write_file/read_file tools \
25             in a fresh sandbox. Each run is scored on OUTCOME (the scorer builds it and gets a \
26             'pong') and on PROCESS (did the agent do all six things the prompt asked?), plus \
27             wall time. RUNS MODEL-AUTHORED SHELL inside the sandbox directory.",
28    duration_hint: "~5 min per iteration",
29    expected_secs: 600,
30    updated: "2026-08-14",
31    needs_confirmation: true,
32    // 2026-09-26: The qwen3.6-35b-a3b entry is the default variant. The
33    // qwen3.6-27b entry is `unmeasured` and has no metric bounds; the
34    // qwen3.8-27b entry is `measured` and has its own.
35    intended_for: Some(crate::benchmark::ModelExpectation {
36        families: &["qwen3.6-35b-a3b", "qwen3.6-27b", "qwen3.8-27b"],
37        note: "This benchmark is defined on the 35B MoE flagship (Qwen3.6-35B-A3B, FP8 or \
38               NVFP4 — the required Gate A subject) and on both dense 27B variants. \
39               Qwen3.6-27B is registered but UNMEASURED: its BENCH.toml entry has no \
40               thresholds, so a run there baselines, it does not gate. Qwen3.8-27B is \
41               MEASURED and gates against its own thresholds. Each variant carries its own \
42               thresholds and serve recipe; any other checkpoint would produce numbers that \
43               compare to nothing.",
44    }),
45    // 2026-09-26: Unless given with `--param`, `wall_budget_s` and
46    // `s_per_turn_budget` take the selected variant's `sum_wall_s` and
47    // `s_per_turn` bounds, when that entry has them.
48    threshold_params: &[
49        ("wall_budget_s", "sum_wall_s"),
50        ("s_per_turn_budget", "s_per_turn"),
51    ],
52    // 2026-09-26: `webserver_ok` and `followed_directions` are correctness
53    // counts, but `wall_budget_s` and `s_per_turn_budget` bound speed, so the
54    // benchmark is classed as Speed.
55    sensitivity: Sensitivity::Speed,
56    ctor: || Box::new(AgenticWebserver::default()),
57};
metrale_bench/benchmarks/agentic/descriptors.rs

metrale_bench/benchmarks/agentic/
descriptors.rs
