metrale_grammar/support/
encoding.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//
3// UTF-8 codepoint encode/decode + hex + Latin-1 — port of the UTF-8
4// half of `cpp/support/encoding.h`. Escape handling lives in the
5// sibling `escape` module.
6//
7// The C++ uses `int32_t` codepoints with negative sentinel values for
8// errors. We keep the same `TCodepoint` alias and `CharHandlingError`
9// values for faithful behavior; decoding APIs additionally expose a
10// Rust-idiomatic `Result`-free `Option` form where useful.
11
12/// A Unicode codepoint, matching C++ `TCodepoint = int32_t`.
13pub type TCodepoint = i32;
14
15/// Error sentinels returned in-band as negative [`TCodepoint`] values,
16/// matching C++ `enum CharHandlingError`.
17pub mod char_handling_error {
18    use super::TCodepoint;
19    /// The UTF-8 string is invalid.
20    pub const INVALID_UTF8: TCodepoint = -10;
21    /// The escape sequence is invalid.
22    pub const INVALID_ESCAPE: TCodepoint = -11;
23    /// The Latin-1 string is invalid.
24    pub const INVALID_LATIN1: TCodepoint = -12;
25}
26
27/// Encode a codepoint into UTF-8 bytes.
28///
29/// Faithful to C++ `CharToUTF8`: it does not validate surrogates and
30/// produces 1–4 bytes for codepoints in `0..=0x10FFFF`. Codepoints
31/// above `0x10FFFF` are debug-checked in C++; here we still encode
32/// using the 4-byte form (the low 21 bits), never panicking.
33pub fn char_to_utf8(codepoint: TCodepoint) -> Vec<u8> {
34    let cp = codepoint as u32;
35    let mut out = Vec::with_capacity(4);
36    if cp <= 0x7F {
37        out.push(cp as u8);
38    } else if cp <= 0x7FF {
39        out.push(0xC0 | ((cp >> 6) & 0x1F) as u8);
40        out.push(0x80 | (cp & 0x3F) as u8);
41    } else if cp <= 0xFFFF {
42        out.push(0xE0 | ((cp >> 12) & 0x0F) as u8);
43        out.push(0x80 | ((cp >> 6) & 0x3F) as u8);
44        out.push(0x80 | (cp & 0x3F) as u8);
45    } else {
46        out.push(0xF0 | ((cp >> 18) & 0x07) as u8);
47        out.push(0x80 | ((cp >> 12) & 0x3F) as u8);
48        out.push(0x80 | ((cp >> 6) & 0x3F) as u8);
49        out.push(0x80 | (cp & 0x3F) as u8);
50    }
51    out
52}
53
54/// Lookup table: number of bytes a UTF-8 sequence starting with each
55/// possible first byte spans, or `-1` for an invalid first byte.
56const UTF8_BYTES: [i8; 256] = build_utf8_bytes();
57
58const fn build_utf8_bytes() -> [i8; 256] {
59    let mut t = [0i8; 256];
60    let mut i = 0;
61    while i < 256 {
62        t[i] = if i < 0x80 {
63            1
64        } else if i < 0xC0 {
65            -1
66        } else if i < 0xE0 {
67            2
68        } else if i < 0xF0 {
69            3
70        } else if i < 0xF8 {
71            4
72        } else {
73            -1
74        };
75        i += 1;
76    }
77    t
78}
79
80/// Mask applied to the first byte to extract its codepoint bits,
81/// indexed by sequence length (`0` slot unused).
82const FIRST_BYTE_MASK: [u8; 5] = [0x00, 0x7F, 0x1F, 0x0F, 0x07];
83
84/// Inspect a UTF-8 first byte.
85///
86/// Returns `(is_valid, total_byte_count, initial_codepoint_bits)`,
87/// matching C++ `HandleUTF8FirstByte`.
88pub fn handle_utf8_first_byte(byte: u8) -> (bool, i32, TCodepoint) {
89    let num_bytes = UTF8_BYTES[byte as usize];
90    if num_bytes == -1 {
91        return (false, 0, 0);
92    }
93    (
94        true,
95        num_bytes as i32,
96        (byte & FIRST_BYTE_MASK[num_bytes as usize]) as TCodepoint,
97    )
98}
99
100/// Decode the first codepoint from a UTF-8 byte slice.
101///
102/// Returns `(codepoint, bytes_consumed)`. On invalid UTF-8 (bad first
103/// byte, missing/invalid continuation byte, or truncated input)
104/// returns `(INVALID_UTF8, 0)` — faithful to C++ `ParseNextUTF8`.
105pub fn parse_next_utf8(utf8: &[u8]) -> (TCodepoint, i32) {
106    if utf8.is_empty() {
107        return (char_handling_error::INVALID_UTF8, 0);
108    }
109    let (mut accepted, num_bytes, mut res) = handle_utf8_first_byte(utf8[0]);
110    if accepted {
111        for i in 1..num_bytes as usize {
112            // C++ treats a NUL byte (string terminator) or any byte
113            // whose top bits are not `10` as invalid. A short slice is
114            // likewise invalid (the C++ string would have terminated).
115            if i >= utf8.len() || utf8[i] == 0 || (utf8[i] & 0xC0) != 0x80 {
116                accepted = false;
117                break;
118            }
119            res = (res << 6) | (utf8[i] & 0x3F) as TCodepoint;
120        }
121    }
122    if !accepted {
123        return (char_handling_error::INVALID_UTF8, 0);
124    }
125    (res, num_bytes)
126}
127
128/// Decode every codepoint in a UTF-8 byte slice.
129///
130/// When `preserve_invalid_bytes` is `false` (the C++ default), an
131/// invalid byte aborts decoding and the result is the single-element
132/// vector `[INVALID_UTF8]`. When `true`, each invalid byte is pushed
133/// as its raw `u8` value (widened to `TCodepoint`) and decoding
134/// continues. Faithful to C++ `ParseUTF8`. Decoding stops at the first
135/// NUL byte, matching the C-string semantics of the original.
136pub fn parse_utf8(utf8: &[u8], preserve_invalid_bytes: bool) -> Vec<TCodepoint> {
137    let mut codepoints = Vec::new();
138    let mut pos = 0;
139    while pos < utf8.len() && utf8[pos] != 0 {
140        let (codepoint, num_bytes) = parse_next_utf8(&utf8[pos..]);
141        if codepoint == char_handling_error::INVALID_UTF8 {
142            if preserve_invalid_bytes {
143                codepoints.push(utf8[pos] as TCodepoint);
144                pos += 1;
145                continue;
146            } else {
147                return vec![char_handling_error::INVALID_UTF8];
148            }
149        }
150        codepoints.push(codepoint);
151        pos += num_bytes as usize;
152    }
153    codepoints
154}
155
156/// Convert a hex digit (`0-9`, `a-f`, `A-F`) to its value, or `-1` if
157/// not a hex digit. Faithful to C++ `HexCharToInt`.
158pub fn hex_char_to_int(c: u8) -> i32 {
159    match c {
160        b'0'..=b'9' => (c - b'0') as i32,
161        b'a'..=b'f' => (c - b'a' + 10) as i32,
162        b'A'..=b'F' => (c - b'A' + 10) as i32,
163        _ => -1,
164    }
165}
166
167/// Convert a Latin-1 string (given as bytes that are themselves valid
168/// UTF-8) into the raw Latin-1 byte sequence.
169///
170/// Every codepoint must be in `0x00..=0xFF`; ASCII passes through and
171/// `0x80..=0xFF` is expected as a 2-byte UTF-8 sequence. Returns
172/// `Err(INVALID_LATIN1)` on any out-of-range or malformed input —
173/// faithful to C++ `Latin1ToBytes`.
174pub fn latin1_to_bytes(latin1: &[u8]) -> Result<Vec<u8>, TCodepoint> {
175    let mut result = Vec::with_capacity(latin1.len());
176    let mut i = 0;
177    while i < latin1.len() {
178        let c1 = latin1[i];
179        if c1 < 0x80 {
180            result.push(c1);
181            i += 1;
182        } else {
183            if i + 1 >= latin1.len() {
184                return Err(char_handling_error::INVALID_LATIN1);
185            }
186            let c2 = latin1[i + 1];
187            if (c2 & 0xC0) != 0x80 {
188                return Err(char_handling_error::INVALID_LATIN1);
189            }
190            let code = (((c1 & 0x1F) as i32) << 6) | (c2 & 0x3F) as i32;
191            if !(0x80..=0xFF).contains(&code) {
192                return Err(char_handling_error::INVALID_LATIN1);
193            }
194            result.push(code as u8);
195            i += 2;
196        }
197    }
198    Ok(result)
199}
200
201/// Convert a raw byte sequence into a Latin-1 string encoded as UTF-8.
202///
203/// Bytes `0x00..=0x7F` pass through; `0x80..=0xFF` are emitted as the
204/// 2-byte UTF-8 encoding of the same codepoint. Faithful to C++
205/// `ByteToLatin1`. Stops at the first NUL byte, matching the original.
206pub fn byte_to_latin1(bytes: &[u8]) -> Vec<u8> {
207    let mut result = Vec::new();
208    for &b in bytes {
209        if b == 0 {
210            break;
211        }
212        if b <= 0x7F {
213            result.push(b);
214        } else {
215            result.push(0xC0 | (b >> 6));
216            result.push(0x80 | (b & 0x3F));
217        }
218    }
219    result
220}
221
222#[cfg(test)]
223mod tests {
224    use super::char_handling_error::*;
225    use super::*;
226
227    #[test]
228    fn encode_one_byte() {
229        assert_eq!(char_to_utf8(0x41), vec![0x41]);
230        assert_eq!(char_to_utf8(0x00), vec![0x00]);
231        assert_eq!(char_to_utf8(0x7F), vec![0x7F]);
232    }
233
234    #[test]
235    fn encode_two_byte() {
236        // U+00A9 ©
237        assert_eq!(char_to_utf8(0xA9), vec![0xC2, 0xA9]);
238        // U+07FF boundary
239        assert_eq!(char_to_utf8(0x7FF), vec![0xDF, 0xBF]);
240    }
241
242    #[test]
243    fn encode_three_byte() {
244        // U+2603 ☃
245        assert_eq!(char_to_utf8(0x2603), vec![0xE2, 0x98, 0x83]);
246        // U+FFFF boundary
247        assert_eq!(char_to_utf8(0xFFFF), vec![0xEF, 0xBF, 0xBF]);
248    }
249
250    #[test]
251    fn encode_four_byte() {
252        // U+1F600 😀
253        assert_eq!(char_to_utf8(0x1F600), vec![0xF0, 0x9F, 0x98, 0x80]);
254        // U+10FFFF max codepoint
255        assert_eq!(char_to_utf8(0x10FFFF), vec![0xF4, 0x8F, 0xBF, 0xBF]);
256    }
257
258    #[test]
259    fn encode_decode_round_trip() {
260        for cp in [0x41, 0xA9, 0x2603, 0x1F600, 0x6D4B, 0x8BD5] {
261            let bytes = char_to_utf8(cp);
262            let (decoded, n) = parse_next_utf8(&bytes);
263            assert_eq!(decoded, cp);
264            assert_eq!(n as usize, bytes.len());
265        }
266    }
267
268    #[test]
269    fn first_byte_classification() {
270        assert_eq!(handle_utf8_first_byte(0x41), (true, 1, 0x41));
271        assert!(handle_utf8_first_byte(0xC2).0);
272        assert_eq!(handle_utf8_first_byte(0xC2).1, 2);
273        assert_eq!(handle_utf8_first_byte(0xE2).1, 3);
274        assert_eq!(handle_utf8_first_byte(0xF0).1, 4);
275        // Continuation byte cannot start a sequence.
276        assert_eq!(handle_utf8_first_byte(0x80), (false, 0, 0));
277        // 0xF8.. is invalid.
278        assert_eq!(handle_utf8_first_byte(0xFF), (false, 0, 0));
279    }
280
281    #[test]
282    fn decode_empty_is_invalid() {
283        assert_eq!(parse_next_utf8(&[]), (INVALID_UTF8, 0));
284    }
285
286    #[test]
287    fn decode_invalid_first_byte() {
288        assert_eq!(parse_next_utf8(&[0x80]), (INVALID_UTF8, 0));
289    }
290
291    #[test]
292    fn decode_bad_continuation() {
293        // 0xC2 followed by 0x20 (not a continuation byte).
294        assert_eq!(parse_next_utf8(&[0xC2, 0x20]), (INVALID_UTF8, 0));
295    }
296
297    #[test]
298    fn decode_truncated_sequence() {
299        // 0xE2 expects 3 bytes but only 1 is present.
300        assert_eq!(parse_next_utf8(&[0xE2]), (INVALID_UTF8, 0));
301    }
302
303    #[test]
304    fn parse_utf8_string() {
305        let s = "UTF-8: © ☃ 😀".as_bytes();
306        let cps = parse_utf8(s, false);
307        assert!(cps.contains(&0xA9));
308        assert!(cps.contains(&0x2603));
309        assert!(cps.contains(&0x1F600));
310    }
311
312    #[test]
313    fn parse_utf8_invalid_aborts() {
314        let cps = parse_utf8(&[0x41, 0xC2, 0x20], false);
315        assert_eq!(cps, vec![INVALID_UTF8]);
316    }
317
318    #[test]
319    fn parse_utf8_preserve_invalid() {
320        let cps = parse_utf8(&[0x41, 0x80, 0x42], true);
321        assert_eq!(cps, vec![0x41, 0x80, 0x42]);
322    }
323
324    #[test]
325    fn parse_utf8_empty() {
326        assert_eq!(parse_utf8(&[], false), Vec::<TCodepoint>::new());
327    }
328
329    #[test]
330    fn parse_utf8_stops_at_nul() {
331        let cps = parse_utf8(&[0x41, 0x00, 0x42], false);
332        assert_eq!(cps, vec![0x41]);
333    }
334
335    #[test]
336    fn hex_digits() {
337        assert_eq!(hex_char_to_int(b'0'), 0);
338        assert_eq!(hex_char_to_int(b'9'), 9);
339        assert_eq!(hex_char_to_int(b'a'), 10);
340        assert_eq!(hex_char_to_int(b'f'), 15);
341        assert_eq!(hex_char_to_int(b'A'), 10);
342        assert_eq!(hex_char_to_int(b'F'), 15);
343        assert_eq!(hex_char_to_int(b'g'), -1);
344        assert_eq!(hex_char_to_int(b' '), -1);
345    }
346
347    #[test]
348    fn latin1_round_trip() {
349        // 0xFF -> two UTF-8 bytes -> back to 0xFF
350        let utf8 = byte_to_latin1(&[0xFF]);
351        assert_eq!(utf8, vec![0xC3, 0xBF]);
352        assert_eq!(latin1_to_bytes(&utf8), Ok(vec![0xFF]));
353    }
354
355    #[test]
356    fn latin1_ascii_passthrough() {
357        assert_eq!(byte_to_latin1(b"Abc"), b"Abc".to_vec());
358        assert_eq!(latin1_to_bytes(b"Abc"), Ok(b"Abc".to_vec()));
359    }
360
361    #[test]
362    fn latin1_invalid_truncated() {
363        assert_eq!(latin1_to_bytes(&[0xC3]), Err(INVALID_LATIN1));
364    }
365
366    #[test]
367    fn latin1_invalid_bad_continuation() {
368        assert_eq!(latin1_to_bytes(&[0xC3, 0x20]), Err(INVALID_LATIN1));
369    }
370
371    #[test]
372    fn latin1_invalid_out_of_range() {
373        // 0xC4 0x80 = U+0100, just above the Latin-1 range 0x00..=0xFF.
374        // (c1 & 0x1F)=0x04, <<6 = 0x100, | (0x80 & 0x3F)=0 -> 0x100.
375        assert_eq!(latin1_to_bytes(&[0xC4, 0x80]), Err(INVALID_LATIN1));
376        // 0xC2 0x40: 0x40 is not a valid continuation byte.
377        assert_eq!(latin1_to_bytes(&[0xC2, 0x40]), Err(INVALID_LATIN1));
378    }
379
380    #[test]
381    fn byte_to_latin1_stops_at_nul() {
382        assert_eq!(byte_to_latin1(&[0x41, 0x00, 0x42]), vec![0x41]);
383    }
384}
metrale_grammar/support/encoding.rs

metrale_grammar/support/
encoding.rs
