metrale_telemetry/
request.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: L5: per-request latency (TTFT, TPOT, end-to-end) and energy.
4//!
5//! Owner: telemetry.
6//! Invariants: a non-finite or negative time records as 0 ns.
7
8use crate::instrument::{Counter, MillijouleHistogram, NanosHistogram};
9
10/// 2026-09-26: A finished request's timing, as the scheduler reports it with
11/// the request's terminal frame. A request that ends in an error frame
12/// reports none.
13#[derive(Clone, Copy, Debug, PartialEq)]
14pub struct RequestTiming {
15    /// 2026-09-26: Request start to decode start.
16    pub ttft_ms: f64,
17    /// 2026-09-26: Decode start to the terminal frame.
18    pub decode_ms: f64,
19    pub output_tokens: u64,
20}
21
22impl RequestTiming {
23    /// 2026-09-26: Request start to the terminal frame: `ttft_ms + decode_ms`, in ns.
24    pub fn e2e_ns(&self) -> u64 {
25        ms_to_ns(self.ttft_ms + self.decode_ms)
26    }
27
28    /// 2026-09-26: Mean time per output token after the first; `None` for a
29    /// request that produced fewer than two tokens.
30    pub fn tpot_ns(&self) -> Option<u64> {
31        (self.output_tokens >= 2)
32            .then(|| ms_to_ns(self.decode_ms / (self.output_tokens - 1) as f64))
33    }
34}
35
36fn ms_to_ns(ms: f64) -> u64 {
37    if ms.is_finite() && ms > 0.0 {
38        (ms * 1e6) as u64
39    } else {
40        0
41    }
42}
43
44#[derive(Debug)]
45pub struct RequestInstruments {
46    pub finished: Counter,
47    /// 2026-09-26: Output tokens of finished requests, as their terminal frames report.
48    pub output_tokens: Counter,
49    pub ttft: NanosHistogram,
50    pub tpot: NanosHistogram,
51    pub e2e: NanosHistogram,
52    pub energy: MillijouleHistogram,
53    /// 2026-09-26: Finished requests charged no energy: the device layer was
54    /// not available, or their window had no ring point or no tokens.
55    pub energy_unattributed: Counter,
56}
57
58impl Default for RequestInstruments {
59    fn default() -> Self {
60        Self::new()
61    }
62}
63
64impl RequestInstruments {
65    pub const fn new() -> Self {
66        Self {
67            finished: Counter::new(),
68            output_tokens: Counter::new(),
69            ttft: NanosHistogram::new(),
70            tpot: NanosHistogram::new(),
71            e2e: NanosHistogram::new(),
72            energy: MillijouleHistogram::new(),
73            energy_unattributed: Counter::new(),
74        }
75    }
76
77    pub(crate) fn record(&self, r: &RequestTiming, energy_mj: Option<u64>) {
78        self.finished.add(1);
79        self.output_tokens.add(r.output_tokens);
80        self.ttft.observe(ms_to_ns(r.ttft_ms));
81        if let Some(t) = r.tpot_ns() {
82            self.tpot.observe(t);
83        }
84        self.e2e.observe(r.e2e_ns());
85        match energy_mj {
86            Some(mj) => self.energy.observe(mj),
87            None => self.energy_unattributed.add(1),
88        }
89    }
90}
91
92#[cfg(test)]
93mod tests {
94    use super::*;
95
96    #[test]
97    fn tpot_excludes_the_first_token_and_needs_two() {
98        let r = RequestTiming {
99            ttft_ms: 100.0,
100            decode_ms: 90.0,
101            output_tokens: 10,
102        };
103        assert_eq!(r.tpot_ns(), Some(10_000_000));
104        assert_eq!(r.e2e_ns(), 190_000_000);
105        let one = RequestTiming {
106            output_tokens: 1,
107            ..r
108        };
109        assert_eq!(one.tpot_ns(), None);
110    }
111
112    #[test]
113    fn a_non_finite_time_records_as_zero_not_a_wrapped_huge_value() {
114        assert_eq!(ms_to_ns(f64::NAN), 0);
115        assert_eq!(ms_to_ns(-5.0), 0);
116        assert_eq!(ms_to_ns(f64::INFINITY), 0);
117    }
118}
metrale_telemetry/request.rs

metrale_telemetry/
request.rs
