metrale_bench/benchmarks/scheduler_equivalence/
unmeasured.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-25: Why a sample is unmeasured: one entry per request that failed or never came back.
4//!
5//! Owner: bench, scheduler-equivalence gate.
6//! Invariants:
7//! - Every sample a cell counts as unmeasured has at least one entry here:
8//!   the reference's failure, the compared leg's failure, or its absence.
9//! - An entry names its leg, lane, concurrency and sample, so a record
10//!   explains its failure without a rerun.
11//! - Pure: no I/O, and nothing here takes part in the verdict.
12
13use std::collections::BTreeMap;
14use std::fmt;
15use std::time::Duration;
16
17use super::compare::{Leg, Pass};
18use super::host::Lane;
19use crate::http::{FailureKind, RequestFailure};
20
21/// 2026-09-25: Every cause class, each a `unmeasured_cause_<class>` metric
22/// even at zero.
23pub const CLASSES: [&str; 10] = [
24    "timeout",
25    "http_4xx",
26    "http_5xx",
27    "http_other",
28    "transport",
29    "malformed",
30    "truncated",
31    "server_error",
32    "missing",
33    "other",
34];
35
36#[derive(Debug, Clone, PartialEq, Eq)]
37pub enum Cause {
38    Failed {
39        failure: RequestFailure,
40        elapsed: Duration,
41    },
42    /// 2026-09-25: The leg holds no reply for the sample, or the leg never ran.
43    Missing,
44}
45
46#[derive(Debug, Clone, PartialEq, Eq)]
47pub struct Unmeasured {
48    pub pass: Pass,
49    pub lane: Lane,
50    pub concurrency: usize,
51    pub sample_id: String,
52    pub cause: Cause,
53}
54
55impl Unmeasured {
56    /// 2026-09-25: One of `CLASSES`.
57    pub fn class(&self) -> &'static str {
58        let Cause::Failed { failure, .. } = &self.cause else {
59            return "missing";
60        };
61        match failure.kind {
62            FailureKind::Timeout => "timeout",
63            FailureKind::Status(c) if (400..500).contains(&c) => "http_4xx",
64            FailureKind::Status(c) if (500..600).contains(&c) => "http_5xx",
65            FailureKind::Status(_) => "http_other",
66            FailureKind::Transport => "transport",
67            FailureKind::Malformed => "malformed",
68            FailureKind::Truncated => "truncated",
69            FailureKind::ServerError => "server_error",
70            FailureKind::Other => "other",
71        }
72    }
73}
74
75/// 2026-09-25: One line, every string quoted, so a body with newlines stays
76/// on its line and the fields stay parseable.
77impl fmt::Display for Unmeasured {
78    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
79        write!(
80            f,
81            "unmeasured: leg={} lane={} C={} sample={:?} cause={}",
82            self.pass.label(),
83            self.lane.label(),
84            self.concurrency,
85            self.sample_id,
86            self.class()
87        )?;
88        match &self.cause {
89            Cause::Missing => write!(f, " error=\"no reply in this leg\""),
90            Cause::Failed { failure, elapsed } => {
91                let status = match failure.kind {
92                    FailureKind::Status(c) => c.to_string(),
93                    _ => "-".to_string(),
94                };
95                write!(
96                    f,
97                    " status={status} finish_reason={} elapsed={:.3}s error={:?} body={:?}",
98                    failure.finish_reason.as_deref().unwrap_or("-"),
99                    elapsed.as_secs_f64(),
100                    failure.message,
101                    failure.body
102                )
103            }
104        }
105    }
106}
107
108/// 2026-09-25: The entries for one cell, walking the reference's samples as
109/// `compare::diff` does. The candidate leg is required, so its absence is an
110/// entry per sample; an absent control leg was not run and is none.
111pub fn collect(reference: &Leg, candidate: Option<&Leg>, control: Option<&Leg>) -> Vec<Unmeasured> {
112    let entry = |pass, sample_id: &str, cause| Unmeasured {
113        pass,
114        lane: reference.lane,
115        concurrency: reference.concurrency,
116        sample_id: sample_id.to_string(),
117        cause,
118    };
119    let mut out: Vec<Unmeasured> = reference
120        .replies
121        .iter()
122        .filter_map(|r| {
123            let failure = r.outcome.as_ref().err()?;
124            Some(entry(Pass::Sync, &r.sample_id, failed(failure, r.elapsed)))
125        })
126        .collect();
127    if candidate.is_none() {
128        out.extend(
129            reference
130                .replies
131                .iter()
132                .map(|r| entry(Pass::Async, &r.sample_id, Cause::Missing)),
133        );
134    }
135    for other in [candidate, control].into_iter().flatten() {
136        for r in &reference.replies {
137            let cause = match other.find(&r.sample_id) {
138                None => Cause::Missing,
139                Some(o) => match &o.outcome {
140                    Ok(_) => continue,
141                    Err(failure) => failed(failure, o.elapsed),
142                },
143            };
144            out.push(entry(other.pass, &r.sample_id, cause));
145        }
146    }
147    out
148}
149
150fn failed(failure: &RequestFailure, elapsed: Duration) -> Cause {
151    Cause::Failed {
152        failure: failure.clone(),
153        elapsed,
154    }
155}
156
157/// 2026-09-25: `"<leg> <class> <n>"` per (leg, class), comma-separated, or
158/// `"none"`. Leg and class order are fixed, so equal inputs read the same.
159pub fn breakdown(entries: &[Unmeasured]) -> String {
160    let mut tally: BTreeMap<(Pass, &str), usize> = BTreeMap::new();
161    for e in entries {
162        *tally.entry((e.pass, e.class())).or_default() += 1;
163    }
164    if tally.is_empty() {
165        return "none".to_string();
166    }
167    tally
168        .iter()
169        .map(|((pass, class), n)| format!("{} {class} {n}", pass.label()))
170        .collect::<Vec<_>>()
171        .join(", ")
172}
173
174/// 2026-09-25: `unmeasured_cause_<class>` for every class. These count
175/// requests, not comparisons, so they need not sum to `unmeasured`.
176pub fn metrics<'a>(entries: impl Iterator<Item = &'a Unmeasured>) -> BTreeMap<String, f64> {
177    let key = |class: &str| format!("unmeasured_cause_{class}");
178    let mut m: BTreeMap<String, f64> = CLASSES.iter().map(|c| (key(c), 0.0)).collect();
179    for e in entries {
180        *m.entry(key(e.class())).or_default() += 1.0;
181    }
182    m
183}
metrale_bench/benchmarks/scheduler_equivalence/unmeasured.rs

metrale_bench/benchmarks/scheduler_equivalence/
unmeasured.rs
