metrale_server/
refusal.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! 2026-09-26: Heuristic refusal detector: a prefix matcher over the assistant's opening text.
4//!
5//! Owner: server API.
6//! Invariants:
7//! - `detect` returns `None` whenever `METRALE_DISABLE_REFUSAL_DETECTION=1`.
8//! - A returned sentence is a leading slice of the input, trimmed, in the
9//!   model's original case.
10//!
11//! It fills `message.refusal` on the blocking chat path and the refusal delta
12//! on the streaming path, both only when no tool call was found. It is not a
13//! safety classifier: it misses refusals that open differently and fires on
14//! content that opens by quoting one. `/v1/moderations` answers 501.
15
16/// 2026-09-26: Prefixes matched against the lowercased first 48 characters of the
17/// message after leading whitespace. Any match counts, so order does not matter.
18const REFUSAL_PREFIXES: &[&str] = &[
19    "i cannot ",
20    "i can't help with ",
21    "i can't assist with ",
22    "i'm not able to ",
23    "i am not able to ",
24    "i'm unable to ",
25    "i am unable to ",
26    "i must decline",
27    "i won't assist",
28    "i will not assist",
29    "i won't help",
30    "i will not help",
31    "sorry, i cannot",
32    "sorry, but i can't",
33    "sorry, but i cannot",
34    "i'm sorry, but i can't",
35    "i'm sorry, but i cannot",
36    "i apologize, but i can't",
37    "i apologize, but i cannot",
38    "as an ai, i cannot",
39    "as an ai, i can't",
40    "as an ai language model, i cannot",
41    "as an ai language model, i can't",
42];
43
44/// 2026-09-26: Returns the refusal sentence when `content` opens with one of the
45/// known patterns, else `None`. The sentence ends at the first `.`, `?` or `!`
46/// within 512 characters, else at the first newline, and is trimmed. When
47/// `METRALE_DISABLE_REFUSAL_DETECTION=1`, always returns `None`.
48pub fn detect(content: &str) -> Option<String> {
49    if std::env::var("METRALE_DISABLE_REFUSAL_DETECTION").as_deref() == Ok("1") {
50        return None;
51    }
52    let trimmed = content.trim_start();
53    if trimmed.is_empty() {
54        return None;
55    }
56    // 2026-09-26: Compare against a lowercase view but return the original-cased
57    // sentence, so the client sees the model's exact words.
58    let head: String = trimmed
59        .chars()
60        .take(48)
61        .flat_map(|c| c.to_lowercase())
62        .collect();
63    let matched = REFUSAL_PREFIXES.iter().any(|p| head.starts_with(p));
64    if !matched {
65        return None;
66    }
67    // 2026-09-26: The first sentence ends at the first terminal punctuation within
68    // 512 characters; with none, fall back to the first line.
69    let end_idx = trimmed
70        .char_indices()
71        .take(512)
72        .find(|(_, c)| matches!(c, '.' | '?' | '!'))
73        .map(|(i, c)| i + c.len_utf8());
74    let sentence = match end_idx {
75        Some(i) => &trimmed[..i],
76        None => trimmed
77            .split_once('\n')
78            .map(|(line, _)| line)
79            .unwrap_or(trimmed),
80    };
81    Some(sentence.trim().to_string())
82}
83
84#[cfg(test)]
85mod tests {
86    use super::*;
87    use std::sync::Mutex;
88
89    // 2026-09-26: Unit tests run in parallel threads of one process, and env vars
90    // are process-wide. `kill_switch_returns_none` sets
91    // METRALE_DISABLE_REFUSAL_DETECTION, so every test here that calls `detect()`
92    // holds this lock.
93    static ENV_LOCK: Mutex<()> = Mutex::new(());
94
95    #[test]
96    fn matches_canonical_refusal() {
97        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
98        let r = detect("I cannot help with that request. Here is why…").unwrap();
99        assert_eq!(r, "I cannot help with that request.");
100    }
101
102    #[test]
103    fn matches_with_leading_whitespace() {
104        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
105        let r = detect("   I'm sorry, but I can't assist with weapons design.").unwrap();
106        assert_eq!(r, "I'm sorry, but I can't assist with weapons design.");
107    }
108
109    #[test]
110    fn mixed_case_matches() {
111        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
112        assert!(detect("As AN ai, I cannot provide that.").is_some());
113    }
114
115    #[test]
116    fn non_refusal_returns_none() {
117        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
118        assert!(detect("Sure, here's how to do that.").is_none());
119        assert!(detect("").is_none());
120        assert!(detect("I can do that for you.").is_none());
121    }
122
123    #[test]
124    fn kill_switch_returns_none() {
125        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
126        // 2026-09-26: SAFETY: serialized by ENV_LOCK against this module's tests only.
127        unsafe {
128            std::env::set_var("METRALE_DISABLE_REFUSAL_DETECTION", "1");
129        }
130        let got = detect("I cannot help with that.");
131        // 2026-09-26: SAFETY: serialized by ENV_LOCK against this module's tests only.
132        unsafe {
133            std::env::remove_var("METRALE_DISABLE_REFUSAL_DETECTION");
134        }
135        assert!(got.is_none());
136    }
137
138    #[test]
139    fn no_terminator_falls_back_to_line() {
140        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
141        let r = detect("I cannot answer that\nnext paragraph").unwrap();
142        assert_eq!(r, "I cannot answer that");
143    }
144}
metrale_server/refusal.rs

metrale_server/
refusal.rs
