metrale_grammar/grammar/functor/
char_range_same_len.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//
3// `add_same_length_range` — the same-byte-length half of the UTF-8
4// character-range FSM helper. Split out of `char_range.rs` to keep each
5// file under the 250-line cap. Port of `AddSameLengthCharacterRange`
6// from `cpp/grammar_functor.cc`.
7
8use crate::fsm::FsmWithStartEnd;
9
10/// Decompose `[min, max]` of packed UTF-8 values of the *same byte
11/// length* into byte-range edges from state `from` to state `to`.
12pub fn add_same_length_range(
13    fsm: &mut FsmWithStartEnd,
14    from: usize,
15    to: usize,
16    mut min: u32,
17    mut max: u32,
18) {
19    let bmin = |v: u32| {
20        [
21            (v & 0xFF) as u8,
22            (v >> 8) as u8,
23            (v >> 16) as u8,
24            (v >> 24) as u8,
25        ]
26    };
27    let mut byte_min = bmin(min);
28    let mut byte_max = bmin(max);
29
30    // ASCII (single byte).
31    if byte_max[1] == 0 {
32        fsm.fsm_mut()
33            .add_edge(from, to, byte_min[0] as i16, byte_max[0] as i16);
34        return;
35    }
36
37    if byte_max[3] != 0 {
38        // 4-byte unicode.
39        if byte_max[3] == byte_min[3] {
40            let tmp = fsm.add_state();
41            fsm.fsm_mut()
42                .add_edge(from, tmp, byte_min[3] as i16, byte_max[3] as i16);
43            min &= 0x00FF_FFFF;
44            max &= 0x00FF_FFFF;
45            add_same_length_range(fsm, tmp, to, min, max);
46            return;
47        }
48        if (min & 0x00FF_FFFF) != 0x808080 {
49            let tmin = fsm.add_state();
50            fsm.fsm_mut()
51                .add_edge(from, tmin, byte_min[3] as i16, byte_min[3] as i16);
52            add_same_length_range(fsm, tmin, to, min & 0x00FF_FFFF, 0x00BF_BFBF);
53        } else {
54            byte_min[3] -= 1;
55        }
56        if (max & 0x00FF_FFFF) != 0xBFBFBF {
57            let tmax = fsm.add_state();
58            fsm.fsm_mut()
59                .add_edge(from, tmax, byte_max[3] as i16, byte_max[3] as i16);
60            add_same_length_range(fsm, tmax, to, 0x0080_8080, max & 0x00FF_FFFF);
61        } else {
62            byte_max[3] += 1;
63        }
64        if byte_max[3] as i16 - byte_min[3] as i16 > 1 {
65            let m1 = fsm.add_state();
66            fsm.fsm_mut()
67                .add_edge(from, m1, byte_min[3] as i16 + 1, byte_max[3] as i16 - 1);
68            let m2 = fsm.add_state();
69            fsm.fsm_mut().add_edge(m1, m2, 0x80, 0xBF);
70            let m3 = fsm.add_state();
71            fsm.fsm_mut().add_edge(m2, m3, 0x80, 0xBF);
72            fsm.fsm_mut().add_edge(m3, to, 0x80, 0xBF);
73        }
74        return;
75    }
76    if byte_max[2] != 0 {
77        // 3-byte unicode.
78        if byte_max[2] == byte_min[2] {
79            let tmp = fsm.add_state();
80            fsm.fsm_mut()
81                .add_edge(from, tmp, byte_min[2] as i16, byte_max[2] as i16);
82            min &= 0x00FFFF;
83            max &= 0x00FFFF;
84            add_same_length_range(fsm, tmp, to, min, max);
85            return;
86        }
87        if (min & 0x00FFFF) != 0x8080 {
88            let tmin = fsm.add_state();
89            fsm.fsm_mut()
90                .add_edge(from, tmin, byte_min[2] as i16, byte_min[2] as i16);
91            add_same_length_range(fsm, tmin, to, min & 0x00FFFF, 0x00BFBF);
92        } else {
93            byte_min[2] -= 1;
94        }
95        if (max & 0x00FFFF) != 0xBFBF {
96            let tmax = fsm.add_state();
97            fsm.fsm_mut()
98                .add_edge(from, tmax, byte_max[2] as i16, byte_max[2] as i16);
99            add_same_length_range(fsm, tmax, to, 0x0080, max & 0x00FFFF);
100        } else {
101            byte_max[2] += 1;
102        }
103        if byte_max[2] as i16 - byte_min[2] as i16 > 1 {
104            let m1 = fsm.add_state();
105            fsm.fsm_mut()
106                .add_edge(from, m1, byte_min[2] as i16 + 1, byte_max[2] as i16 - 1);
107            let m2 = fsm.add_state();
108            fsm.fsm_mut().add_edge(m1, m2, 0x80, 0xBF);
109            fsm.fsm_mut().add_edge(m2, to, 0x80, 0xBF);
110        }
111        return;
112    }
113
114    // 2-byte unicode.
115    if byte_max[1] == byte_min[1] {
116        let tmp = fsm.add_state();
117        fsm.fsm_mut()
118            .add_edge(from, tmp, byte_min[1] as i16, byte_max[1] as i16);
119        min &= 0x00FF;
120        max &= 0x00FF;
121        add_same_length_range(fsm, tmp, to, min, max);
122        return;
123    }
124    if (min & 0x00FF) != 0x80 {
125        let tmin = fsm.add_state();
126        fsm.fsm_mut()
127            .add_edge(from, tmin, byte_min[1] as i16, byte_min[1] as i16);
128        add_same_length_range(fsm, tmin, to, min & 0x00FF, 0x00BF);
129    } else {
130        byte_min[1] -= 1;
131    }
132    if (max & 0x00FF) != 0xBF {
133        let tmax = fsm.add_state();
134        fsm.fsm_mut()
135            .add_edge(from, tmax, byte_max[1] as i16, byte_max[1] as i16);
136        add_same_length_range(fsm, tmax, to, 0x0080, max & 0x00FF);
137    } else {
138        byte_max[1] += 1;
139    }
140    if byte_max[1] as i16 - byte_min[1] as i16 > 1 {
141        let m1 = fsm.add_state();
142        fsm.fsm_mut()
143            .add_edge(from, m1, byte_min[1] as i16 + 1, byte_max[1] as i16 - 1);
144        fsm.fsm_mut().add_edge(m1, to, 0x80, 0xBF);
145    }
146}
metrale_grammar/grammar/functor/char_range_same_len.rs

metrale_grammar/grammar/functor/
char_range_same_len.rs
