metrale_bench/benchmarks/contamination/
prompts.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: The probe corpus: prompts built so that contamination is
4//! detectable.
5//!
6//! - Each prompt carries a canary and asks the model to echo it, so the canary
7//!   is in both the prompt tokens and the generated tokens. A foreign canary
8//!   in a reply is leakage on its own evidence
9//!   ([`super::transcript::Transcript::carries_foreign_canary`]).
10//! - The canaries share no `-`-separated component and neither contains the
11//!   other (tested below).
12//! - The prompts are identical up to the canary: the shared preamble is a
13//!   prefix-cache collision surface, and the text right after it is the
14//!   canary sentence, so resuming the wrong continuation at that boundary
15//!   emits the foreign canary.
16//! - The topics differ (lighthouse, water mill), so a leak that mangles the
17//!   canary still diverges from the solo reference (`Class::Diverged`).
18//! - The driver sends temperature 0, and the prompts ask for a fixed shape
19//!   (code, three sentences, code).
20//!
21//! Owner: bench (contamination).
22//! Invariants: every canary occurs in its own prompt's tail and in no other
23//! prompt (tested below).
24
25/// 2026-09-26: A prompt and the canary that must not appear in any other
26/// probe's reply.
27pub struct Probe {
28    pub name: &'static str,
29    pub canary: &'static str,
30    /// 2026-09-26: The probe-specific instruction; [`Probe::prompt`] prefixes
31    /// the shared preamble.
32    tail: &'static str,
33}
34
35impl Probe {
36    /// 2026-09-26: Shared preamble followed by this probe's tail. The preamble
37    /// is one const, so the prompts are identical up to the tail.
38    pub fn prompt(&self) -> String {
39        format!("{PREAMBLE}{}", self.tail)
40    }
41}
42
43const PREAMBLE: &str = "You are a precise assistant taking part in a determinism audit. \
44                        Follow the instructions exactly and do not add anything else. ";
45
46pub const PROBES: [Probe; 2] = [
47    Probe {
48        name: "A",
49        canary: "XK-AZURE-HERON-41",
50        tail: "Your reference code is XK-AZURE-HERON-41. Begin your reply with the \
51               reference code on its own line, then explain in exactly three short \
52               sentences how a lighthouse warns ships at night, then end with the \
53               reference code again on its own line.",
54    },
55    Probe {
56        name: "B",
57        canary: "QJ-CRIMSON-OTTER-77",
58        tail: "Your reference code is QJ-CRIMSON-OTTER-77. Begin your reply with the \
59               reference code on its own line, then explain in exactly three short \
60               sentences how a water mill grinds grain into flour, then end with the \
61               reference code again on its own line.",
62    },
63];
64
65/// 2026-09-26: The canaries in probe order: [`super::score::score`] reads the
66/// canary at a probe's index as that probe's own.
67pub fn canaries() -> Vec<String> {
68    PROBES.iter().map(|p| p.canary.to_string()).collect()
69}
70
71#[cfg(test)]
72mod tests {
73    use super::*;
74    use crate::benchmarks::contamination::transcript::Transcript;
75
76    #[test]
77    fn every_canary_is_unique_to_its_probe() {
78        for (i, a) in PROBES.iter().enumerate() {
79            assert!(
80                a.prompt().contains(a.canary),
81                "probe {} does not carry its own canary",
82                a.name
83            );
84            for (j, b) in PROBES.iter().enumerate() {
85                if i == j {
86                    continue;
87                }
88                assert!(
89                    !b.prompt().contains(a.canary),
90                    "probe {}'s prompt contains probe {}'s canary — a clean run \
91                     would read as contaminated",
92                    b.name,
93                    a.name
94                );
95                assert!(
96                    !a.canary.contains(b.canary) && !b.canary.contains(a.canary),
97                    "canaries {} / {} overlap as substrings",
98                    a.canary,
99                    b.canary
100                );
101                let a_parts: std::collections::BTreeSet<_> = a.canary.split('-').collect();
102                let b_parts: std::collections::BTreeSet<_> = b.canary.split('-').collect();
103                assert!(
104                    a_parts.is_disjoint(&b_parts),
105                    "canaries {} / {} share lexical components: {:?}",
106                    a.canary,
107                    b.canary,
108                    a_parts.intersection(&b_parts).collect::<Vec<_>>()
109                );
110            }
111        }
112    }
113
114    #[test]
115    fn prompts_share_the_preamble_and_diverge_at_the_canary() {
116        let full: Vec<String> = PROBES.iter().map(Probe::prompt).collect();
117        let canary_offsets: Vec<usize> = PROBES
118            .iter()
119            .zip(&full)
120            .map(|(probe, prompt)| prompt.find(probe.canary).expect("canary in own prompt"))
121            .collect();
122        assert_eq!(canary_offsets[0], canary_offsets[1]);
123        let boundary = canary_offsets[0];
124        assert_eq!(&full[0][..boundary], &full[1][..boundary]);
125        assert_eq!(
126            &full[0][..boundary],
127            format!("{PREAMBLE}Your reference code is ")
128        );
129        for p in &PROBES {
130            assert!(
131                p.prompt().starts_with(PREAMBLE),
132                "probe {} lost the shared preamble",
133                p.name
134            );
135            assert!(
136                !PREAMBLE.contains(p.canary) && p.tail.contains(p.canary),
137                "probe {}'s canary must live in the divergent tail, not the \
138                 shared span",
139                p.name
140            );
141        }
142    }
143
144    #[test]
145    fn names_are_distinct_and_canaries_ride_in_probe_order() {
146        let mut names = std::collections::BTreeSet::new();
147        for p in &PROBES {
148            assert!(names.insert(p.name), "duplicate probe name {}", p.name);
149        }
150        assert_eq!(
151            canaries(),
152            PROBES
153                .iter()
154                .map(|p| p.canary.to_string())
155                .collect::<Vec<_>>(),
156            "Legs indexes canaries by prompt position; order is load-bearing"
157        );
158    }
159
160    #[test]
161    fn the_core_detector_fires_on_these_canaries() {
162        let cans = canaries();
163        let all: Vec<&str> = cans.iter().map(String::as_str).collect();
164        let clean = Transcript {
165            text: format!(
166                "{}\nA lighthouse shines a rotating beam. Ships see it from afar. \
167                 They steer clear of the rocks.\n{}",
168                PROBES[0].canary, PROBES[0].canary
169            ),
170            ..Default::default()
171        };
172        assert_eq!(
173            clean.carries_foreign_canary(PROBES[0].canary, &all),
174            None,
175            "a probe's own canary must not read as leakage"
176        );
177        let leaked = Transcript {
178            text: format!("{}\nA lighthouse shines...", PROBES[1].canary),
179            ..Default::default()
180        };
181        assert_eq!(
182            leaked.carries_foreign_canary(PROBES[0].canary, &all),
183            Some(PROBES[1].canary),
184            "B's canary in A's reply must be attributed to B"
185        );
186    }
187}
metrale_bench/benchmarks/contamination/prompts.rs

metrale_bench/benchmarks/contamination/
prompts.rs
