metrale_bench/benchmarks/agentic/
norm.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Normalising `bash` tool output so that identical work reads
4//! identically: progress lines are dropped, and timestamps, dates, durations,
5//! pids and `ps` bookkeeping columns become placeholders. Every tool output is
6//! fed back to the model, so a build that reports `in 0.00s` on one run and
7//! `in 1.23s` on the next gives two greedy runs different contexts.
8//!
9//! Each rule is anchored on a shape only tool bookkeeping has: a cargo status
10//! word at the start of a line, a `time(1)` row, curl's progress meter, a
11//! timestamp. Compiler diagnostics pass through. Outside a `fuser` line, the
12//! pid rules rewrite only runs of four or more digits, so `(exit status: 101)`
13//! and `2 passed` survive. A port survives when it is glued to `/tcp`, `:` or
14//! `.`, or follows the word `port`.
15//!
16//! Only `bash` results are normalised (`agent_shell.rs` `run_shell`), before
17//! truncation. `read`, `grep` and `glob` results are not, so an `edit` built
18//! from a `read` matches the file. No rule can fix a `ps` COMMAND column that
19//! shows another process on the box, or timing that changes what a command
20//! prints.
21//!
22//! Owner: bench, agentic.
23//! Invariants:
24//! - `normalize` is idempotent: every placeholder it writes is digit-free.
25//! - It drops only progress lines, and reorders only runs of libtest result
26//!   lines; every other line keeps its place.
27
28/// 2026-09-26: Prefixes (after leading whitespace) of cargo progress lines,
29/// which differ between a cold build and a warm one. `Finished`, which says
30/// whether the build worked, is not among them.
31const PROGRESS: [&str; 7] = [
32    "Compiling ",
33    "Fresh ",
34    "Downloading ",
35    "Downloaded ",
36    "Updating ",
37    "Locking ",
38    "Blocking waiting",
39];
40
41const MONTHS: [&str; 12] = [
42    "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
43];
44
45/// 2026-09-26: Two-letter units first, so `ms` is tried before `m`.
46const UNITS: [&str; 6] = ["ns", "µs", "us", "ms", "s", "m"];
47
48/// 2026-09-26: Normalise one shell result. Idempotent: every placeholder it
49/// writes is digit-free, so no rule matches its own output.
50pub fn normalize(text: &str) -> String {
51    let mut lines: Vec<String> = text
52        .split('\n')
53        .filter(|line| !is_noise(line))
54        .map(scrub)
55        .collect();
56    sort_test_results(&mut lines);
57    lines.join("\n")
58}
59
60/// 2026-09-26: Sort each run of consecutive `test <name> ... <result>` lines,
61/// which libtest prints in the order tests finish. The `test result:` summary
62/// and a failing test's `---- <name> stdout ----` block do not have that shape
63/// and keep their place.
64fn sort_test_results(lines: &mut [String]) {
65    let mut i = 0;
66    while i < lines.len() {
67        if !is_test_result(&lines[i]) {
68            i += 1;
69            continue;
70        }
71        let mut end = i;
72        while end < lines.len() && is_test_result(&lines[end]) {
73            end += 1;
74        }
75        lines[i..end].sort();
76        i = end;
77    }
78}
79
80fn is_test_result(line: &str) -> bool {
81    line.starts_with("test ") && line.contains(" ... ")
82}
83
84/// 2026-09-26: A progress line: a [`PROGRESS`] prefix or a curl meter row.
85fn is_noise(line: &str) -> bool {
86    let head = line.trim_start();
87    PROGRESS.iter().any(|p| head.starts_with(p))
88        // 2026-09-26: curl's progress meter: its two header rows, and every row
89        // with a `--:--:--` clock.
90        || line.contains("--:--:--")
91        || head.starts_with("% Total")
92        || head.starts_with("Dload")
93}
94
95fn scrub(line: &str) -> String {
96    let line = replace_spans(line);
97    let line = trailing_duration(&line);
98    let line = time_builtin(&line);
99    pids(&line)
100}
101
102/// 2026-09-26: Rewrite every timestamp- or date-shaped span, at word
103/// boundaries only, so digits inside an identifier (a cargo metadata hash, a
104/// version) are never taken for one.
105fn replace_spans(line: &str) -> String {
106    let mut out = String::with_capacity(line.len());
107    let mut i = 0;
108    while i < line.len() {
109        let rest = &line[i..];
110        let boundary = !line[..i].chars().next_back().is_some_and(identifier_char);
111        if boundary && let Some((len, replacement)) = span_at(rest) {
112            let ends_at_boundary = !rest[len..].chars().next().is_some_and(identifier_char);
113            if ends_at_boundary {
114                out.push_str(replacement);
115                i += len;
116                continue;
117            }
118        }
119        let c = rest.chars().next().unwrap_or('\0');
120        out.push(c);
121        i += c.len_utf8();
122    }
123    out
124}
125
126fn identifier_char(c: char) -> bool {
127    c.is_ascii_alphanumeric() || c == '_'
128}
129
130fn span_at(s: &str) -> Option<(usize, &'static str)> {
131    if let Some(n) = timestamp(s) {
132        return Some((n, "<timestamp>"));
133    }
134    if let Some(n) = listing_date(s) {
135        return Some((n, "<date>"));
136    }
137    // 2026-09-26: `curl: (7) Failed to connect to localhost port 3001 after 0
138    // ms: …`: the port stays, the duration after `after` does not.
139    if let Some(rest) = s.strip_prefix("after")
140        && let Some(n) = duration(rest.as_bytes(), 0)
141    {
142        return Some((5 + n, "after <elapsed>"));
143    }
144    None
145}
146
147/// 2026-09-26: `2026-08-06T21:09:12.345Z`, and the same with a space for the
148/// `T`. A bare date with no clock is left alone: it could be data.
149fn timestamp(s: &str) -> Option<usize> {
150    let b = s.as_bytes();
151    let mut i = digits(b, 0, 4, 4)?;
152    i = byte(b, i, b'-')?;
153    i = digits(b, i, 2, 2)?;
154    i = byte(b, i, b'-')?;
155    i = digits(b, i, 2, 2)?;
156    if !matches!(b.get(i), Some(b'T') | Some(b' ')) {
157        return None;
158    }
159    let mut j = clock(b, i + 1)?;
160    if b.get(j) == Some(&b'.') {
161        j = digits(b, j + 1, 1, 9)?;
162    }
163    match b.get(j) {
164        Some(b'Z') => j += 1,
165        Some(b'+') | Some(b'-') => {
166            if let Some(k) = digits(b, j + 1, 2, 2)
167                .and_then(|k| byte(b, k, b':'))
168                .and_then(|k| digits(b, k, 2, 2))
169            {
170                j = k;
171            }
172        }
173        _ => {}
174    }
175    Some(j)
176}
177
178/// 2026-09-26: A month name, day, and clock or year: `Aug  6 21:09`,
179/// `Aug  6 21:09:12`, `Aug  6  2025` (`ls -l`, `date`).
180fn listing_date(s: &str) -> Option<usize> {
181    let month = MONTHS.iter().find(|m| s.starts_with(**m))?;
182    let b = s.as_bytes();
183    let mut i = spaces(b, month.len())?;
184    i = digits(b, i, 1, 2)?;
185    let j = spaces(b, i)?;
186    clock(b, j).or_else(|| digits(b, j, 4, 4))
187}
188
189fn clock(b: &[u8], i: usize) -> Option<usize> {
190    let mut j = digits(b, i, 2, 2)?;
191    j = byte(b, j, b':')?;
192    j = digits(b, j, 2, 2)?;
193    if b.get(j) == Some(&b':') {
194        j = digits(b, j + 1, 2, 2)?;
195    }
196    Some(j)
197}
198
199/// 2026-09-26: A line that ends in `… in <duration>`: cargo's `Finished` line,
200/// libtest's `test result: … finished in 0.00s`. Only a whole duration at the
201/// end of the line matches, which keeps prose containing " in " intact.
202fn trailing_duration(line: &str) -> String {
203    let trimmed = line.trim_end();
204    let Some(cut) = trimmed.rfind(" in ") else {
205        return line.to_string();
206    };
207    let tail = &trimmed[cut + 4..];
208    match whole_duration(tail) {
209        true => format!("{} in <elapsed>", &trimmed[..cut]),
210        false => line.to_string(),
211    }
212}
213
214/// 2026-09-26: `time(1)`'s three rows: `real\t0m1.234s`, `user …`, `sys …`.
215fn time_builtin(line: &str) -> String {
216    let head = line.trim_start();
217    let Some(word) = ["real", "user", "sys"]
218        .iter()
219        .find(|w| head.starts_with(**w))
220    else {
221        return line.to_string();
222    };
223    let rest = &head[word.len()..];
224    let value = rest.trim_start();
225    if value.len() == rest.len() || !whole_duration(value.trim_end()) {
226        return line.to_string();
227    }
228    format!("{}{word}\t<elapsed>", &line[..line.len() - head.len()])
229}
230
231fn whole_duration(s: &str) -> bool {
232    duration(s.as_bytes(), 0) == Some(s.len()) && !s.is_empty()
233}
234
235/// 2026-09-26: One or more `<number><unit>` groups, each optionally preceded
236/// by a space: `0.42s`, `1m 03s`, `0m1.234s`, `0 ms`.
237fn duration(b: &[u8], start: usize) -> Option<usize> {
238    let (mut i, mut any) = (start, false);
239    loop {
240        let mut j = i;
241        if b.get(j) == Some(&b' ') {
242            j += 1;
243        }
244        let Some(n) = number(b, j) else { break };
245        let mut k = n;
246        if b.get(k) == Some(&b' ') {
247            k += 1;
248        }
249        let Some(end) = unit(b, k) else { break };
250        i = end;
251        any = true;
252    }
253    any.then_some(i)
254}
255
256/// 2026-09-26: A unit counts only when no letter follows it, so `3 more` is not
257/// 3 minutes and `2 seconds` is not `2 s` + `econds`.
258fn unit(b: &[u8], i: usize) -> Option<usize> {
259    let rest = std::str::from_utf8(b.get(i..)?).ok()?;
260    let u = UNITS.iter().find(|u| rest.starts_with(**u))?;
261    let end = i + u.len();
262    match b.get(end) {
263        Some(c) if c.is_ascii_alphabetic() => None,
264        _ => Some(end),
265    }
266}
267
268fn number(b: &[u8], i: usize) -> Option<usize> {
269    let j = digits(b, i, 1, 12)?;
270    match b.get(j) {
271        Some(b'.') => digits(b, j + 1, 1, 12),
272        _ => Some(j),
273    }
274}
275
276/// 2026-09-26: Rewrite process ids.
277///
278/// [`ps_line`] and [`fuser_line`] are tried first. `fuser`'s `3001/tcp:  12345`
279/// keeps the port and rewrites the free-standing numbers after the colon.
280/// Otherwise, runs of four to seven digits are rewritten on a line that
281/// consists only of such numbers (`echo $!`, `pgrep`, `lsof -t`) or that names
282/// a pid with a [`NAMES_A_PID`] word (`kill: (12345) - No such process`).
283///
284/// A four-digit number alone on a line (`… | wc -c`, or a port) is rewritten
285/// too. `ps` output is covered only by [`ps_line`].
286fn pids(line: &str) -> String {
287    if let Some(rewritten) = ps_line(line).or_else(|| fuser_line(line)) {
288        return rewritten;
289    }
290    // 2026-09-26: A line that already carries `<pid>` has been through here;
291    // matching the word inside it would let a second pass rewrite digits the
292    // first pass kept.
293    let named = !line.contains("<pid>")
294        && line
295            .split(|c: char| !c.is_ascii_alphabetic())
296            .any(|w| NAMES_A_PID.iter().any(|k| w.eq_ignore_ascii_case(k)));
297    match named || bare_numbers(line) {
298        true => digit_runs(line, 4, 7),
299        false => line.to_string(),
300    }
301}
302
303/// 2026-09-26: Words that mark a line as naming a pid. Matched as whole words,
304/// so `Processing` and `skill` do not match.
305const NAMES_A_PID: [&str; 7] = [
306    "kill",
307    "killed",
308    "killing",
309    "pid",
310    "pids",
311    "process",
312    "processes",
313];
314
315/// 2026-09-26: `ps aux` output with the pid, %CPU, %MEM, VSZ, RSS, START and
316/// TIME columns replaced; USER, TTY, STAT and COMMAND are kept.
317///
318/// Recognised by shape, because `| grep` strips the header: eleven or more
319/// fields, an all-digit pid, two `N.N` percentages, two integer sizes, and a
320/// colon-separated TIME in the tenth field.
321fn ps_line(line: &str) -> Option<String> {
322    let f: Vec<&str> = line.split_whitespace().collect();
323    if f.len() < 11 {
324        return None;
325    }
326    let int = |s: &str| !s.is_empty() && s.bytes().all(|c| c.is_ascii_digit());
327    let decimal = |s: &str| s.split_once('.').is_some_and(|(a, b)| int(a) && int(b));
328    let elapsed = |s: &str| s.contains(':') && s.split(':').all(int);
329    let shaped =
330        int(f[1]) && decimal(f[2]) && decimal(f[3]) && int(f[4]) && int(f[5]) && elapsed(f[9]);
331    shaped.then(|| {
332        format!(
333            "{} <pid> <cpu> <mem> <vsz> <rss> {} {} <start> <time> {}",
334            f[0],
335            f[6],
336            f[7],
337            f[10..].join(" ")
338        )
339    })
340}
341
342fn fuser_line(line: &str) -> Option<String> {
343    let head = line.trim_start();
344    let n = digits(head.as_bytes(), 0, 1, 5)?;
345    let rest = ["/tcp:", "/udp:"]
346        .iter()
347        .find_map(|p| head[n..].strip_prefix(*p))?;
348    let keep = &line[..line.len() - rest.len()];
349    Some(format!("{keep}{}", digit_runs(rest, 1, 9)))
350}
351
352fn bare_numbers(line: &str) -> bool {
353    let mut fields = line.split_whitespace().peekable();
354    fields.peek().is_some()
355        && fields.all(|f| (4..=7).contains(&f.len()) && f.bytes().all(|c| c.is_ascii_digit()))
356}
357
358/// 2026-09-26: Replace every free-standing run of `min..=max` digits.
359///
360/// A run that touches a letter, a digit or a [`GLUED`] byte on either side is
361/// part of something else and is kept: `pingpong-9a8b7c6d` (a cargo metadata
362/// hash), `0.0.0.0:3001` and `3001/tcp` (an address and a port). So is a run
363/// right after the word `port` ([`after_port_word`]).
364fn digit_runs(s: &str, min: usize, max: usize) -> String {
365    let (b, mut out, mut i) = (s.as_bytes(), String::with_capacity(s.len()), 0);
366    while i < s.len() {
367        if b[i].is_ascii_digit() {
368            let end = digits(b, i, 1, usize::MAX).unwrap_or(i);
369            let glued_before = i > 0 && glued(b[i - 1]);
370            let glued_after = b.get(end).copied().is_some_and(glued);
371            let free = !(glued_before || glued_after || after_port_word(s, i));
372            match free && (min..=max).contains(&(end - i)) {
373                true => out.push_str("<pid>"),
374                false => out.push_str(&s[i..end]),
375            }
376            i = end;
377            continue;
378        }
379        let c = s[i..].chars().next().unwrap_or('\0');
380        out.push(c);
381        i += c.len_utf8();
382    }
383    out
384}
385
386/// 2026-09-26: Is the number at `i` right after the word `port` (ignoring
387/// spaces, tabs, `=` and `:`)? Then it is a port: `# kill whatever is on port
388/// 3001`.
389fn after_port_word(s: &str, i: usize) -> bool {
390    let head = s[..i].trim_end_matches([' ', '\t', '=', ':']);
391    head.rsplit(|c: char| !c.is_ascii_alphabetic())
392        .next()
393        .is_some_and(|w| w.eq_ignore_ascii_case("port"))
394}
395
396/// 2026-09-26: Non-alphanumeric neighbours that make a number part of a larger
397/// token.
398const GLUED: [u8; 4] = [b'.', b':', b'/', b'-'];
399
400fn glued(c: u8) -> bool {
401    c.is_ascii_alphanumeric() || GLUED.contains(&c)
402}
403
404fn digits(b: &[u8], i: usize, min: usize, max: usize) -> Option<usize> {
405    let mut end = i;
406    while end < b.len() && b[end].is_ascii_digit() && end - i < max {
407        end += 1;
408    }
409    (end - i >= min).then_some(end)
410}
411
412fn byte(b: &[u8], i: usize, want: u8) -> Option<usize> {
413    (b.get(i) == Some(&want)).then_some(i + 1)
414}
415
416fn spaces(b: &[u8], i: usize) -> Option<usize> {
417    let mut end = i;
418    while b.get(end) == Some(&b' ') {
419        end += 1;
420    }
421    (end > i).then_some(end)
422}
423
424#[cfg(test)]
425#[path = "norm_tests.rs"]
426mod tests;
metrale_bench/benchmarks/agentic/norm.rs

metrale_bench/benchmarks/agentic/
norm.rs
