spark_server/
refusal.rs

1// SPDX-License-Identifier: AGPL-3.0-only
2
3//! Heuristic refusal classifier.
4//!
5//! Populates `message.refusal` on the blocking chat-completion path so
6//! safety-aware clients (OpenAI Python SDK, Vercel AI SDK) that branch on
7//! `message.refusal != null` see the expected shape. Atlas does **not**
8//! train safety behavior into its models — this detector only recognizes
9//! the text the underlying model emits when it declines to answer.
10//!
11//! Honest scope: a prefix-matcher, not a safety classifier. It catches the
12//! common refusal openings ("I cannot help with…", "I'm sorry, but I
13//! can't…", "As an AI…") that instruction-tuned models produce. It will
14//! miss subtle refusals and will false-positive on content that quotes a
15//! refusal. Clients that need real safety-classification should run their
16//! own moderation pass — `/v1/moderations` is a 501 stub on this server.
17//!
18//! Set `ATLAS_DISABLE_REFUSAL_DETECTION=1` to force `refusal: None` on all
19//! responses, matching pre-PR-4 behavior byte-for-byte.
20
21/// Prefix patterns matched case-insensitively against the stripped leading
22/// text of the assistant message. Order matters only for determinism — the
23/// first match wins.
24const REFUSAL_PREFIXES: &[&str] = &[
25    "i cannot ",
26    "i can't help with ",
27    "i can't assist with ",
28    "i'm not able to ",
29    "i am not able to ",
30    "i'm unable to ",
31    "i am unable to ",
32    "i must decline",
33    "i won't assist",
34    "i will not assist",
35    "i won't help",
36    "i will not help",
37    "sorry, i cannot",
38    "sorry, but i can't",
39    "sorry, but i cannot",
40    "i'm sorry, but i can't",
41    "i'm sorry, but i cannot",
42    "i apologize, but i can't",
43    "i apologize, but i cannot",
44    "as an ai, i cannot",
45    "as an ai, i can't",
46    "as an ai language model, i cannot",
47    "as an ai language model, i can't",
48];
49
50/// Returns the refusal sentence when `content` opens with one of the known
51/// patterns, else `None`. The returned sentence is the first sentence
52/// (truncated at `.`, `?`, or `!`) with trailing whitespace trimmed. When
53/// the kill-switch env var is set, always returns `None`.
54pub fn detect(content: &str) -> Option<String> {
55    if std::env::var("ATLAS_DISABLE_REFUSAL_DETECTION").as_deref() == Ok("1") {
56        return None;
57    }
58    let trimmed = content.trim_start();
59    if trimmed.is_empty() {
60        return None;
61    }
62    // Compare against prefixes using a lowercase view but return the
63    // original-cased sentence so the client sees the model's exact words.
64    let head: String = trimmed
65        .chars()
66        .take(48)
67        .flat_map(|c| c.to_lowercase())
68        .collect();
69    let matched = REFUSAL_PREFIXES.iter().any(|p| head.starts_with(p));
70    if !matched {
71        return None;
72    }
73    // First sentence ends at the first terminal punctuation. If none is
74    // present within a reasonable bound, fall back to the first line.
75    let end_idx = trimmed
76        .char_indices()
77        .take(512)
78        .find(|(_, c)| matches!(c, '.' | '?' | '!'))
79        .map(|(i, c)| i + c.len_utf8());
80    let sentence = match end_idx {
81        Some(i) => &trimmed[..i],
82        None => trimmed
83            .split_once('\n')
84            .map(|(line, _)| line)
85            .unwrap_or(trimmed),
86    };
87    Some(sentence.trim().to_string())
88}
89
90#[cfg(test)]
91mod tests {
92    use super::*;
93    use std::sync::Mutex;
94
95    // Cargo runs unit tests in parallel threads within a single binary, and
96    // env vars are process-wide. `kill_switch_returns_none` mutates
97    // ATLAS_DISABLE_REFUSAL_DETECTION, so every test that calls `detect()`
98    // must hold this lock to avoid observing a transient kill-switch state.
99    static ENV_LOCK: Mutex<()> = Mutex::new(());
100
101    #[test]
102    fn matches_canonical_refusal() {
103        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
104        let r = detect("I cannot help with that request. Here is why…").unwrap();
105        assert_eq!(r, "I cannot help with that request.");
106    }
107
108    #[test]
109    fn matches_with_leading_whitespace() {
110        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
111        let r = detect("   I'm sorry, but I can't assist with weapons design.").unwrap();
112        assert_eq!(r, "I'm sorry, but I can't assist with weapons design.");
113    }
114
115    #[test]
116    fn mixed_case_matches() {
117        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
118        assert!(detect("As AN ai, I cannot provide that.").is_some());
119    }
120
121    #[test]
122    fn non_refusal_returns_none() {
123        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
124        assert!(detect("Sure, here's how to do that.").is_none());
125        assert!(detect("").is_none());
126        assert!(detect("I can do that for you.").is_none());
127    }
128
129    #[test]
130    fn kill_switch_returns_none() {
131        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
132        // SAFETY: serialized via ENV_LOCK across this module's tests.
133        unsafe {
134            std::env::set_var("ATLAS_DISABLE_REFUSAL_DETECTION", "1");
135        }
136        let got = detect("I cannot help with that.");
137        // SAFETY: serialized via ENV_LOCK across this module's tests.
138        unsafe {
139            std::env::remove_var("ATLAS_DISABLE_REFUSAL_DETECTION");
140        }
141        assert!(got.is_none());
142    }
143
144    #[test]
145    fn no_terminator_falls_back_to_line() {
146        let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
147        let r = detect("I cannot answer that\nnext paragraph").unwrap();
148        assert_eq!(r, "I cannot answer that");
149    }
150}
spark_server/refusal.rs

spark_server/
refusal.rs
