diff --git a/Cargo.lock b/Cargo.lock index 73afeda8a18..3a0392b188e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3722,6 +3722,7 @@ name = "rustpython-unicode" version = "0.5.0" dependencies = [ "icu_casemap", + "icu_locale", "icu_normalizer", "icu_properties", "rustpython-wtf8", @@ -3749,9 +3750,6 @@ dependencies = [ "glob", "half", "hex", - "icu_casemap", - "icu_locale", - "icu_properties", "indexmap", "is-macro", "itertools 0.15.0", @@ -3793,7 +3791,6 @@ dependencies = [ "timsort", "wasm-bindgen", "widestring", - "writeable", ] [[package]] diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs index 5deeab67eb6..ca7303a2a7f 100644 --- a/crates/sre_engine/src/string.rs +++ b/crates/sre_engine/src/string.rs @@ -399,40 +399,8 @@ pub(crate) fn is_uni_digit(ch: u32) -> bool { #[inline] pub(crate) fn is_uni_space(ch: u32) -> bool { - // TODO: check with cpython - is_space(ch) - || matches!( - ch, - 0x0009 - | 0x000A - | 0x000B - | 0x000C - | 0x000D - | 0x001C - | 0x001D - | 0x001E - | 0x001F - | 0x0020 - | 0x0085 - | 0x00A0 - | 0x1680 - | 0x2000 - | 0x2001 - | 0x2002 - | 0x2003 - | 0x2004 - | 0x2005 - | 0x2006 - | 0x2007 - | 0x2008 - | 0x2009 - | 0x200A - | 0x2028 - | 0x2029 - | 0x202F - | 0x205F - | 0x3000 - ) + // SRE_UNI_IS_SPACE is Py_UNICODE_ISSPACE. + char::try_from(ch).is_ok_and(rustpython_unicode::classify::is_space) } #[inline] @@ -457,13 +425,13 @@ pub(crate) fn is_uni_word(ch: u32) -> bool { #[inline] #[must_use] pub fn lower_unicode(ch: u32) -> u32 { - // TODO: check with cpython - char::try_from(ch).map_or(ch, |x| x.to_lowercase().next().unwrap() as u32) + // SRE_UNI_LOWER is Py_UNICODE_TOLOWER, the simple one-to-one mapping. + char::try_from(ch).map_or(ch, |x| rustpython_unicode::case::simple_lowercase(x) as u32) } #[inline] #[must_use] pub fn upper_unicode(ch: u32) -> u32 { - // TODO: check with cpython - char::try_from(ch).map_or(ch, |x| x.to_uppercase().next().unwrap() as u32) + // SRE_UNI_UPPER is Py_UNICODE_TOUPPER, the simple one-to-one mapping. + char::try_from(ch).map_or(ch, |x| rustpython_unicode::case::simple_uppercase(x) as u32) } diff --git a/crates/unicode/Cargo.toml b/crates/unicode/Cargo.toml index 678305ee09e..52ab05f4d1f 100644 --- a/crates/unicode/Cargo.toml +++ b/crates/unicode/Cargo.toml @@ -12,6 +12,7 @@ rust-version = { workspace = true } rustpython-wtf8 = { workspace = true } icu_casemap = { workspace = true } +icu_locale = { workspace = true } icu_properties = { workspace = true } icu_normalizer = { workspace = true } unicode_names2 = { workspace = true } diff --git a/crates/unicode/src/case.rs b/crates/unicode/src/case.rs index 872d0b29ac1..0d74133399a 100644 --- a/crates/unicode/src/case.rs +++ b/crates/unicode/src/case.rs @@ -1,19 +1,92 @@ -//! Case folding for Python `str.casefold`. +//! Case mapping, case folding, and casing predicates for Python string casing. //! -//! Lower, upper, and title casing of `str` objects stay with the runtime -//! because they iterate the string with special final-sigma handling. Case -//! folding has no such context dependence, so it lives here and is shared with -//! other runtimes. +//! Code-point mappings (`simple_*`) return a single `char` and back the SRE +//! engine's `IGNORECASE` handling. String-level helpers (`capitalize`, `title`, +//! `swapcase`, `casefold`) implement the full, context-sensitive mappings used +//! by `str` methods and pass lone surrogates through unchanged. The casing +//! predicates expose the derived properties that `str.islower`/`isupper`/ +//! `istitle` need. +//! +//! Plain `str.lower`/`str.upper` have no such context beyond the final-sigma +//! rule that `str::to_lowercase` already applies, so they stay on +//! `rustpython_wtf8::Wtf8::to_lowercase`/`to_uppercase` rather than being +//! duplicated here. + +// spell-checker:ignore ΟΔΟΣ Οδος use alloc::{ string::{String, ToString}, vec::Vec, }; -use icu_casemap::CaseMapper; -use rustpython_wtf8::{Wtf8, Wtf8Buf, Wtf8Chunk}; +use icu_casemap::{CaseMapper, TitlecaseMapper}; +use icu_locale::LanguageIdentifier; +use icu_properties::props::{ + BinaryProperty, CaseIgnorable, Cased, EnumeratedProperty, GeneralCategory, Lowercase, Uppercase, +}; +use rustpython_wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Chunk}; use writeable::Writeable; +// Code-point mappings + +/// Simple (one-to-one) lowercase mapping of `c` (`Py_UNICODE_TOLOWER`). +#[must_use] +pub fn simple_lowercase(c: char) -> char { + CaseMapper::new().simple_lowercase(c) +} + +/// Simple (one-to-one) uppercase mapping of `c` (`Py_UNICODE_TOUPPER`). +#[must_use] +pub fn simple_uppercase(c: char) -> char { + CaseMapper::new().simple_uppercase(c) +} + +/// Simple (one-to-one) titlecase mapping of `c` (`Py_UNICODE_TOTITLE`). +#[must_use] +pub fn simple_titlecase(c: char) -> char { + CaseMapper::new().simple_titlecase(c) +} + +/// Simple (one-to-one) case fold of `c`. +#[must_use] +pub fn simple_fold(c: char) -> char { + CaseMapper::new().simple_fold(c) +} + +// Casing predicates + +/// Whether `c` has the `Lowercase` property. +#[must_use] +pub fn is_lowercase(c: char) -> bool { + Lowercase::for_char(c) +} + +/// Whether `c` has the `Uppercase` property. +#[must_use] +pub fn is_uppercase(c: char) -> bool { + Uppercase::for_char(c) +} + +/// Whether `c` is a titlecase letter (general category `Lt`). +#[must_use] +pub fn is_titlecase(c: char) -> bool { + GeneralCategory::for_char(c) == GeneralCategory::TitlecaseLetter +} + +/// Whether `c` has the `Cased` property. +#[must_use] +pub fn is_cased(c: char) -> bool { + Cased::for_char(c) +} + +/// Whether `c` has the `Case_Ignorable` property. +#[must_use] +pub fn is_case_ignorable(c: char) -> bool { + CaseIgnorable::for_char(c) +} + +// String-level mappings + /// Full Unicode case fold of `text` (`str.casefold`). #[must_use] pub fn casefold_str(text: &str) -> String { @@ -23,29 +96,204 @@ pub fn casefold_str(text: &str) -> String { /// Full Unicode case fold of `text`, passing lone surrogates through unchanged. #[must_use] pub fn casefold_wtf8(text: &Wtf8) -> Wtf8Buf { + map_wtf8(text, |s, out| { + CaseMapper::new() + .fold(s) + .write_to(out) + .expect("writing to an in-memory buffer cannot fail"); + }) +} + +/// Capitalize `text` (`str.capitalize`): titlecase the first cased character, +/// lowercase the rest, with final-sigma context. +#[must_use] +pub fn capitalize_str(text: &str) -> String { + let mut out = Vec::with_capacity(text.len()); + capitalize_utf8(text, &mut FmtWriter(&mut out)); + // SAFETY: capitalize_utf8 only appends valid UTF-8. + unsafe { String::from_utf8_unchecked(out) } +} + +/// Capitalize `text`, passing lone surrogates through unchanged. +/// +/// Only the first character of the whole string is titlecased; every later +/// character (including the first of a run that follows a lone surrogate) is +/// lowercased. +#[must_use] +pub fn capitalize_wtf8(text: &Wtf8) -> Wtf8Buf { let mut out = Vec::with_capacity(text.len()); - let mapper = CaseMapper::new(); + let mut first = true; for chunk in text.chunks() { match chunk { Wtf8Chunk::Utf8(s) => { - mapper - .fold(s) - .write_to(&mut FmtWriter(&mut out)) - .expect("writing to an in-memory buffer cannot fail"); + let mut writer = FmtWriter(&mut out); + if first { + capitalize_utf8(s, &mut writer); + first = false; + } else { + for (i, ch) in s.char_indices() { + lowercase_or_sigma(ch, s, i, &mut writer); + } + } } Wtf8Chunk::Surrogate(c) => { - let mut buf = Wtf8Buf::new(); - buf.push(c); - out.extend_from_slice(buf.as_bytes()); + first = false; + push_surrogate(&mut out, c); } } } // SAFETY: - // * CaseMapper only produces valid UTF-8. + // * capitalize_utf8 / lowercase_or_sigma only append valid UTF-8. // * Surrogates are appended as valid WTF-8 (encoded via Wtf8Buf::push). unsafe { Wtf8Buf::from_bytes_unchecked(out) } } +/// Title case `text` (`str.title`). +#[must_use] +pub fn title_str(text: &str) -> String { + let mut out = Vec::with_capacity(text.len()); + titlecase_string(text, &mut FmtWriter(&mut out)); + // SAFETY: titlecase_string only appends valid UTF-8. + unsafe { String::from_utf8_unchecked(out) } +} + +/// Title case `text`, passing lone surrogates through unchanged. +#[must_use] +pub fn title_wtf8(text: &Wtf8) -> Wtf8Buf { + map_wtf8(text, titlecase_string) +} + +/// Swap the case of every character in `text` (`str.swapcase`). +#[must_use] +pub fn swapcase_str(text: &str) -> String { + let mut out = Vec::with_capacity(text.len()); + swapcase_utf8(text, &mut FmtWriter(&mut out)); + // SAFETY: swapcase_utf8 only appends valid UTF-8. + unsafe { String::from_utf8_unchecked(out) } +} + +/// Swap the case of every character in `text`, passing lone surrogates through. +#[must_use] +pub fn swapcase_wtf8(text: &Wtf8) -> Wtf8Buf { + map_wtf8(text, swapcase_utf8) +} + +// Internal helpers + +/// Run `f` over each valid UTF-8 run of `text`, appending the mapped output and +/// carrying lone surrogates through unchanged. +fn map_wtf8(text: &Wtf8, f: impl Fn(&str, &mut FmtWriter<'_>)) -> Wtf8Buf { + let mut out = Vec::with_capacity(text.len()); + for chunk in text.chunks() { + match chunk { + Wtf8Chunk::Utf8(s) => f(s, &mut FmtWriter(&mut out)), + Wtf8Chunk::Surrogate(c) => push_surrogate(&mut out, c), + } + } + // SAFETY: + // * `f` only appends valid UTF-8. + // * Surrogates are appended as valid WTF-8 (encoded via Wtf8Buf::push). + unsafe { Wtf8Buf::from_bytes_unchecked(out) } +} + +/// Append a lone surrogate to `out` as valid WTF-8 bytes. +fn push_surrogate(out: &mut Vec, c: CodePoint) { + let mut buf = Wtf8Buf::new(); + buf.push(c); + out.extend_from_slice(buf.as_bytes()); +} + +fn capitalize_utf8(s: &str, out: &mut FmtWriter<'_>) { + let mut chars = s.char_indices(); + if let Some((first_pos, first_ch)) = chars.next() { + let first = &s[..first_pos + first_ch.len_utf8()]; + titlecase_segment(first, out); + } + for (i, ch) in chars { + lowercase_or_sigma(ch, s, i, out); + } +} + +/// Title case a string following CPython conventions. +/// +/// The first character of each run of cased characters is title cased and the +/// rest are lowercased; a new run starts after any non-cased character (digits, +/// whitespace, punctuation, etc.). +/// "123abc" -> "123Abc" +/// "123abc456def" -> "123Abc456Def" +/// "123 abc" -> "123 Abc" +fn titlecase_string(s: &str, out: &mut FmtWriter<'_>) { + let mut previous_is_cased = false; + for (i, ch) in s.char_indices() { + if previous_is_cased { + lowercase_or_sigma(ch, s, i, out); + } else { + titlecase_segment(&s[i..i + ch.len_utf8()], out); + } + + previous_is_cased = is_cased(ch); + } +} + +fn titlecase_segment(s: &str, out: &mut FmtWriter<'_>) { + TitlecaseMapper::new() + .titlecase_segment(s, &LanguageIdentifier::UNKNOWN, Default::default()) + .write_to(out) + .expect("writing to an in-memory buffer cannot fail"); +} + +fn lowercase_or_sigma(ch: char, s: &str, i: usize, out: &mut FmtWriter<'_>) { + let sigma = 'Σ'; + if ch == sigma { + push_char(handle_capital_sigma(s, i), out); + } else { + for ch in ch.to_lowercase() { + push_char(ch, out); + } + } +} + +// Handle context-sensitive sigma. +// +// Sigma is handled as a special case. This is more efficient than using icu4x +// to scan the entire string with CaseMapper because CaseMapper would allocate +// to produce a new string. +fn handle_capital_sigma(s: &str, i: usize) -> char { + let (left, rest) = s.split_at(i); + let right = &rest['Σ'.len_utf8()..]; + + // Check if any chars before or after sigma are cased. + let before = left + .chars() + .rev() + .find(|&ch| !is_case_ignorable(ch)) + .is_some_and(is_cased); + let after = right + .chars() + .find(|&ch| !is_case_ignorable(ch)) + .is_some_and(is_cased); + if before && !after { 'ς' } else { 'σ' } +} + +fn swapcase_utf8(s: &str, out: &mut FmtWriter<'_>) { + for (i, ch) in s.char_indices() { + if ch.is_uppercase() { + lowercase_or_sigma(ch, s, i, out); + } else if ch.is_lowercase() { + for ch in ch.to_uppercase() { + push_char(ch, out); + } + } else { + push_char(ch, out); + } + } +} + +fn push_char(ch: char, out: &mut FmtWriter<'_>) { + let mut buf = [0u8; 4]; + out.0.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes()); +} + /// Adapter so `icu`'s `Writeable` output can be appended to a byte buffer. struct FmtWriter<'a>(&'a mut Vec); @@ -58,7 +306,12 @@ impl core::fmt::Write for FmtWriter<'_> { #[cfg(test)] mod tests { - use super::casefold_str; + use rustpython_wtf8::{CodePoint, Wtf8Buf}; + + use super::{ + capitalize_str, casefold_str, is_case_ignorable, is_cased, is_lowercase, is_titlecase, + is_uppercase, simple_lowercase, simple_uppercase, swapcase_str, title_str, title_wtf8, + }; #[test] fn casefold_full_mappings() { @@ -66,4 +319,49 @@ mod tests { assert_eq!(casefold_str("ß"), "ss"); assert_eq!(casefold_str("Σ"), "σ"); } + + #[test] + fn simple_mappings_are_one_to_one() { + // ß has no simple uppercase mapping, so it stays unchanged (unlike the + // full mapping "SS"). + assert_eq!(simple_uppercase('ß'), 'ß'); + assert_eq!(simple_uppercase('a'), 'A'); + assert_eq!(simple_lowercase('A'), 'a'); + // Dž (U+01C5) simple-titlecases to itself but upper/lowercases away. + assert_eq!(simple_uppercase('Dž'), 'DŽ'); + assert_eq!(simple_lowercase('Dž'), 'dž'); + } + + #[test] + fn casing_predicates() { + assert!(is_lowercase('a')); + assert!(!is_lowercase('A')); + assert!(is_uppercase('A')); + assert!(is_titlecase('Dž')); + assert!(!is_titlecase('D')); + assert!(is_cased('a') && is_cased('A')); + assert!(!is_cased('1')); + assert!(is_case_ignorable('\'')); + assert!(!is_case_ignorable('a')); + } + + #[test] + fn capitalize_final_sigma() { + // Final sigma at end of a cased run becomes ς. + assert_eq!(capitalize_str("ΟΔΟΣ"), "Οδος"); + assert_eq!(title_str("hello world"), "Hello World"); + assert_eq!(swapcase_str("Hello"), "hELLO"); + } + + #[test] + fn wtf8_passes_surrogates_through() { + let mut buf = Wtf8Buf::from("ab cd"); + buf.push(CodePoint::from_u32(0xD800).unwrap()); + let titled = title_wtf8(&buf); + assert!(titled.code_points().any(|c| c.to_u32() == 0xD800)); + assert_eq!( + titled.code_points().next().and_then(|c| c.to_char()), + Some('A') + ); + } } diff --git a/crates/unicode/tests/data/cpython3.14_mappings.txt b/crates/unicode/tests/data/cpython3.14_mappings.txt new file mode 100644 index 00000000000..f686c5fd15a --- /dev/null +++ b/crates/unicode/tests/data/cpython3.14_mappings.txt @@ -0,0 +1,2 @@ +# unidata_version 16.0.0 +tolower 41:61,42:62,43:63,44:64,45:65,46:66,47:67,48:68,49:69,4A:6A,4B:6B,4C:6C,4D:6D,4E:6E,4F:6F,50:70,51:71,52:72,53:73,54:74,55:75,56:76,57:77,58:78,59:79,5A:7A,C0:E0,C1:E1,C2:E2,C3:E3,C4:E4,C5:E5,C6:E6,C7:E7,C8:E8,C9:E9,CA:EA,CB:EB,CC:EC,CD:ED,CE:EE,CF:EF,D0:F0,D1:F1,D2:F2,D3:F3,D4:F4,D5:F5,D6:F6,D8:F8,D9:F9,DA:FA,DB:FB,DC:FC,DD:FD,DE:FE,100:101,102:103,104:105,106:107,108:109,10A:10B,10C:10D,10E:10F,110:111,112:113,114:115,116:117,118:119,11A:11B,11C:11D,11E:11F,120:121,122:123,124:125,126:127,128:129,12A:12B,12C:12D,12E:12F,130:69,132:133,134:135,136:137,139:13A,13B:13C,13D:13E,13F:140,141:142,143:144,145:146,147:148,14A:14B,14C:14D,14E:14F,150:151,152:153,154:155,156:157,158:159,15A:15B,15C:15D,15E:15F,160:161,162:163,164:165,166:167,168:169,16A:16B,16C:16D,16E:16F,170:171,172:173,174:175,176:177,178:FF,179:17A,17B:17C,17D:17E,181:253,182:183,184:185,186:254,187:188,189:256,18A:257,18B:18C,18E:1DD,18F:259,190:25B,191:192,193:260,194:263,196:269,197:268,198:199,19C:26F,19D:272,19F:275,1A0:1A1,1A2:1A3,1A4:1A5,1A6:280,1A7:1A8,1A9:283,1AC:1AD,1AE:288,1AF:1B0,1B1:28A,1B2:28B,1B3:1B4,1B5:1B6,1B7:292,1B8:1B9,1BC:1BD,1C4:1C6,1C5:1C6,1C7:1C9,1C8:1C9,1CA:1CC,1CB:1CC,1CD:1CE,1CF:1D0,1D1:1D2,1D3:1D4,1D5:1D6,1D7:1D8,1D9:1DA,1DB:1DC,1DE:1DF,1E0:1E1,1E2:1E3,1E4:1E5,1E6:1E7,1E8:1E9,1EA:1EB,1EC:1ED,1EE:1EF,1F1:1F3,1F2:1F3,1F4:1F5,1F6:195,1F7:1BF,1F8:1F9,1FA:1FB,1FC:1FD,1FE:1FF,200:201,202:203,204:205,206:207,208:209,20A:20B,20C:20D,20E:20F,210:211,212:213,214:215,216:217,218:219,21A:21B,21C:21D,21E:21F,220:19E,222:223,224:225,226:227,228:229,22A:22B,22C:22D,22E:22F,230:231,232:233,23A:2C65,23B:23C,23D:19A,23E:2C66,241:242,243:180,244:289,245:28C,246:247,248:249,24A:24B,24C:24D,24E:24F,370:371,372:373,376:377,37F:3F3,386:3AC,388:3AD,389:3AE,38A:3AF,38C:3CC,38E:3CD,38F:3CE,391:3B1,392:3B2,393:3B3,394:3B4,395:3B5,396:3B6,397:3B7,398:3B8,399:3B9,39A:3BA,39B:3BB,39C:3BC,39D:3BD,39E:3BE,39F:3BF,3A0:3C0,3A1:3C1,3A3:3C3,3A4:3C4,3A5:3C5,3A6:3C6,3A7:3C7,3A8:3C8,3A9:3C9,3AA:3CA,3AB:3CB,3CF:3D7,3D8:3D9,3DA:3DB,3DC:3DD,3DE:3DF,3E0:3E1,3E2:3E3,3E4:3E5,3E6:3E7,3E8:3E9,3EA:3EB,3EC:3ED,3EE:3EF,3F4:3B8,3F7:3F8,3F9:3F2,3FA:3FB,3FD:37B,3FE:37C,3FF:37D,400:450,401:451,402:452,403:453,404:454,405:455,406:456,407:457,408:458,409:459,40A:45A,40B:45B,40C:45C,40D:45D,40E:45E,40F:45F,410:430,411:431,412:432,413:433,414:434,415:435,416:436,417:437,418:438,419:439,41A:43A,41B:43B,41C:43C,41D:43D,41E:43E,41F:43F,420:440,421:441,422:442,423:443,424:444,425:445,426:446,427:447,428:448,429:449,42A:44A,42B:44B,42C:44C,42D:44D,42E:44E,42F:44F,460:461,462:463,464:465,466:467,468:469,46A:46B,46C:46D,46E:46F,470:471,472:473,474:475,476:477,478:479,47A:47B,47C:47D,47E:47F,480:481,48A:48B,48C:48D,48E:48F,490:491,492:493,494:495,496:497,498:499,49A:49B,49C:49D,49E:49F,4A0:4A1,4A2:4A3,4A4:4A5,4A6:4A7,4A8:4A9,4AA:4AB,4AC:4AD,4AE:4AF,4B0:4B1,4B2:4B3,4B4:4B5,4B6:4B7,4B8:4B9,4BA:4BB,4BC:4BD,4BE:4BF,4C0:4CF,4C1:4C2,4C3:4C4,4C5:4C6,4C7:4C8,4C9:4CA,4CB:4CC,4CD:4CE,4D0:4D1,4D2:4D3,4D4:4D5,4D6:4D7,4D8:4D9,4DA:4DB,4DC:4DD,4DE:4DF,4E0:4E1,4E2:4E3,4E4:4E5,4E6:4E7,4E8:4E9,4EA:4EB,4EC:4ED,4EE:4EF,4F0:4F1,4F2:4F3,4F4:4F5,4F6:4F7,4F8:4F9,4FA:4FB,4FC:4FD,4FE:4FF,500:501,502:503,504:505,506:507,508:509,50A:50B,50C:50D,50E:50F,510:511,512:513,514:515,516:517,518:519,51A:51B,51C:51D,51E:51F,520:521,522:523,524:525,526:527,528:529,52A:52B,52C:52D,52E:52F,531:561,532:562,533:563,534:564,535:565,536:566,537:567,538:568,539:569,53A:56A,53B:56B,53C:56C,53D:56D,53E:56E,53F:56F,540:570,541:571,542:572,543:573,544:574,545:575,546:576,547:577,548:578,549:579,54A:57A,54B:57B,54C:57C,54D:57D,54E:57E,54F:57F,550:580,551:581,552:582,553:583,554:584,555:585,556:586,10A0:2D00,10A1:2D01,10A2:2D02,10A3:2D03,10A4:2D04,10A5:2D05,10A6:2D06,10A7:2D07,10A8:2D08,10A9:2D09,10AA:2D0A,10AB:2D0B,10AC:2D0C,10AD:2D0D,10AE:2D0E,10AF:2D0F,10B0:2D10,10B1:2D11,10B2:2D12,10B3:2D13,10B4:2D14,10B5:2D15,10B6:2D16,10B7:2D17,10B8:2D18,10B9:2D19,10BA:2D1A,10BB:2D1B,10BC:2D1C,10BD:2D1D,10BE:2D1E,10BF:2D1F,10C0:2D20,10C1:2D21,10C2:2D22,10C3:2D23,10C4:2D24,10C5:2D25,10C7:2D27,10CD:2D2D,13A0:AB70,13A1:AB71,13A2:AB72,13A3:AB73,13A4:AB74,13A5:AB75,13A6:AB76,13A7:AB77,13A8:AB78,13A9:AB79,13AA:AB7A,13AB:AB7B,13AC:AB7C,13AD:AB7D,13AE:AB7E,13AF:AB7F,13B0:AB80,13B1:AB81,13B2:AB82,13B3:AB83,13B4:AB84,13B5:AB85,13B6:AB86,13B7:AB87,13B8:AB88,13B9:AB89,13BA:AB8A,13BB:AB8B,13BC:AB8C,13BD:AB8D,13BE:AB8E,13BF:AB8F,13C0:AB90,13C1:AB91,13C2:AB92,13C3:AB93,13C4:AB94,13C5:AB95,13C6:AB96,13C7:AB97,13C8:AB98,13C9:AB99,13CA:AB9A,13CB:AB9B,13CC:AB9C,13CD:AB9D,13CE:AB9E,13CF:AB9F,13D0:ABA0,13D1:ABA1,13D2:ABA2,13D3:ABA3,13D4:ABA4,13D5:ABA5,13D6:ABA6,13D7:ABA7,13D8:ABA8,13D9:ABA9,13DA:ABAA,13DB:ABAB,13DC:ABAC,13DD:ABAD,13DE:ABAE,13DF:ABAF,13E0:ABB0,13E1:ABB1,13E2:ABB2,13E3:ABB3,13E4:ABB4,13E5:ABB5,13E6:ABB6,13E7:ABB7,13E8:ABB8,13E9:ABB9,13EA:ABBA,13EB:ABBB,13EC:ABBC,13ED:ABBD,13EE:ABBE,13EF:ABBF,13F0:13F8,13F1:13F9,13F2:13FA,13F3:13FB,13F4:13FC,13F5:13FD,1C89:1C8A,1C90:10D0,1C91:10D1,1C92:10D2,1C93:10D3,1C94:10D4,1C95:10D5,1C96:10D6,1C97:10D7,1C98:10D8,1C99:10D9,1C9A:10DA,1C9B:10DB,1C9C:10DC,1C9D:10DD,1C9E:10DE,1C9F:10DF,1CA0:10E0,1CA1:10E1,1CA2:10E2,1CA3:10E3,1CA4:10E4,1CA5:10E5,1CA6:10E6,1CA7:10E7,1CA8:10E8,1CA9:10E9,1CAA:10EA,1CAB:10EB,1CAC:10EC,1CAD:10ED,1CAE:10EE,1CAF:10EF,1CB0:10F0,1CB1:10F1,1CB2:10F2,1CB3:10F3,1CB4:10F4,1CB5:10F5,1CB6:10F6,1CB7:10F7,1CB8:10F8,1CB9:10F9,1CBA:10FA,1CBD:10FD,1CBE:10FE,1CBF:10FF,1E00:1E01,1E02:1E03,1E04:1E05,1E06:1E07,1E08:1E09,1E0A:1E0B,1E0C:1E0D,1E0E:1E0F,1E10:1E11,1E12:1E13,1E14:1E15,1E16:1E17,1E18:1E19,1E1A:1E1B,1E1C:1E1D,1E1E:1E1F,1E20:1E21,1E22:1E23,1E24:1E25,1E26:1E27,1E28:1E29,1E2A:1E2B,1E2C:1E2D,1E2E:1E2F,1E30:1E31,1E32:1E33,1E34:1E35,1E36:1E37,1E38:1E39,1E3A:1E3B,1E3C:1E3D,1E3E:1E3F,1E40:1E41,1E42:1E43,1E44:1E45,1E46:1E47,1E48:1E49,1E4A:1E4B,1E4C:1E4D,1E4E:1E4F,1E50:1E51,1E52:1E53,1E54:1E55,1E56:1E57,1E58:1E59,1E5A:1E5B,1E5C:1E5D,1E5E:1E5F,1E60:1E61,1E62:1E63,1E64:1E65,1E66:1E67,1E68:1E69,1E6A:1E6B,1E6C:1E6D,1E6E:1E6F,1E70:1E71,1E72:1E73,1E74:1E75,1E76:1E77,1E78:1E79,1E7A:1E7B,1E7C:1E7D,1E7E:1E7F,1E80:1E81,1E82:1E83,1E84:1E85,1E86:1E87,1E88:1E89,1E8A:1E8B,1E8C:1E8D,1E8E:1E8F,1E90:1E91,1E92:1E93,1E94:1E95,1E9E:DF,1EA0:1EA1,1EA2:1EA3,1EA4:1EA5,1EA6:1EA7,1EA8:1EA9,1EAA:1EAB,1EAC:1EAD,1EAE:1EAF,1EB0:1EB1,1EB2:1EB3,1EB4:1EB5,1EB6:1EB7,1EB8:1EB9,1EBA:1EBB,1EBC:1EBD,1EBE:1EBF,1EC0:1EC1,1EC2:1EC3,1EC4:1EC5,1EC6:1EC7,1EC8:1EC9,1ECA:1ECB,1ECC:1ECD,1ECE:1ECF,1ED0:1ED1,1ED2:1ED3,1ED4:1ED5,1ED6:1ED7,1ED8:1ED9,1EDA:1EDB,1EDC:1EDD,1EDE:1EDF,1EE0:1EE1,1EE2:1EE3,1EE4:1EE5,1EE6:1EE7,1EE8:1EE9,1EEA:1EEB,1EEC:1EED,1EEE:1EEF,1EF0:1EF1,1EF2:1EF3,1EF4:1EF5,1EF6:1EF7,1EF8:1EF9,1EFA:1EFB,1EFC:1EFD,1EFE:1EFF,1F08:1F00,1F09:1F01,1F0A:1F02,1F0B:1F03,1F0C:1F04,1F0D:1F05,1F0E:1F06,1F0F:1F07,1F18:1F10,1F19:1F11,1F1A:1F12,1F1B:1F13,1F1C:1F14,1F1D:1F15,1F28:1F20,1F29:1F21,1F2A:1F22,1F2B:1F23,1F2C:1F24,1F2D:1F25,1F2E:1F26,1F2F:1F27,1F38:1F30,1F39:1F31,1F3A:1F32,1F3B:1F33,1F3C:1F34,1F3D:1F35,1F3E:1F36,1F3F:1F37,1F48:1F40,1F49:1F41,1F4A:1F42,1F4B:1F43,1F4C:1F44,1F4D:1F45,1F59:1F51,1F5B:1F53,1F5D:1F55,1F5F:1F57,1F68:1F60,1F69:1F61,1F6A:1F62,1F6B:1F63,1F6C:1F64,1F6D:1F65,1F6E:1F66,1F6F:1F67,1F88:1F80,1F89:1F81,1F8A:1F82,1F8B:1F83,1F8C:1F84,1F8D:1F85,1F8E:1F86,1F8F:1F87,1F98:1F90,1F99:1F91,1F9A:1F92,1F9B:1F93,1F9C:1F94,1F9D:1F95,1F9E:1F96,1F9F:1F97,1FA8:1FA0,1FA9:1FA1,1FAA:1FA2,1FAB:1FA3,1FAC:1FA4,1FAD:1FA5,1FAE:1FA6,1FAF:1FA7,1FB8:1FB0,1FB9:1FB1,1FBA:1F70,1FBB:1F71,1FBC:1FB3,1FC8:1F72,1FC9:1F73,1FCA:1F74,1FCB:1F75,1FCC:1FC3,1FD8:1FD0,1FD9:1FD1,1FDA:1F76,1FDB:1F77,1FE8:1FE0,1FE9:1FE1,1FEA:1F7A,1FEB:1F7B,1FEC:1FE5,1FF8:1F78,1FF9:1F79,1FFA:1F7C,1FFB:1F7D,1FFC:1FF3,2126:3C9,212A:6B,212B:E5,2132:214E,2160:2170,2161:2171,2162:2172,2163:2173,2164:2174,2165:2175,2166:2176,2167:2177,2168:2178,2169:2179,216A:217A,216B:217B,216C:217C,216D:217D,216E:217E,216F:217F,2183:2184,24B6:24D0,24B7:24D1,24B8:24D2,24B9:24D3,24BA:24D4,24BB:24D5,24BC:24D6,24BD:24D7,24BE:24D8,24BF:24D9,24C0:24DA,24C1:24DB,24C2:24DC,24C3:24DD,24C4:24DE,24C5:24DF,24C6:24E0,24C7:24E1,24C8:24E2,24C9:24E3,24CA:24E4,24CB:24E5,24CC:24E6,24CD:24E7,24CE:24E8,24CF:24E9,2C00:2C30,2C01:2C31,2C02:2C32,2C03:2C33,2C04:2C34,2C05:2C35,2C06:2C36,2C07:2C37,2C08:2C38,2C09:2C39,2C0A:2C3A,2C0B:2C3B,2C0C:2C3C,2C0D:2C3D,2C0E:2C3E,2C0F:2C3F,2C10:2C40,2C11:2C41,2C12:2C42,2C13:2C43,2C14:2C44,2C15:2C45,2C16:2C46,2C17:2C47,2C18:2C48,2C19:2C49,2C1A:2C4A,2C1B:2C4B,2C1C:2C4C,2C1D:2C4D,2C1E:2C4E,2C1F:2C4F,2C20:2C50,2C21:2C51,2C22:2C52,2C23:2C53,2C24:2C54,2C25:2C55,2C26:2C56,2C27:2C57,2C28:2C58,2C29:2C59,2C2A:2C5A,2C2B:2C5B,2C2C:2C5C,2C2D:2C5D,2C2E:2C5E,2C2F:2C5F,2C60:2C61,2C62:26B,2C63:1D7D,2C64:27D,2C67:2C68,2C69:2C6A,2C6B:2C6C,2C6D:251,2C6E:271,2C6F:250,2C70:252,2C72:2C73,2C75:2C76,2C7E:23F,2C7F:240,2C80:2C81,2C82:2C83,2C84:2C85,2C86:2C87,2C88:2C89,2C8A:2C8B,2C8C:2C8D,2C8E:2C8F,2C90:2C91,2C92:2C93,2C94:2C95,2C96:2C97,2C98:2C99,2C9A:2C9B,2C9C:2C9D,2C9E:2C9F,2CA0:2CA1,2CA2:2CA3,2CA4:2CA5,2CA6:2CA7,2CA8:2CA9,2CAA:2CAB,2CAC:2CAD,2CAE:2CAF,2CB0:2CB1,2CB2:2CB3,2CB4:2CB5,2CB6:2CB7,2CB8:2CB9,2CBA:2CBB,2CBC:2CBD,2CBE:2CBF,2CC0:2CC1,2CC2:2CC3,2CC4:2CC5,2CC6:2CC7,2CC8:2CC9,2CCA:2CCB,2CCC:2CCD,2CCE:2CCF,2CD0:2CD1,2CD2:2CD3,2CD4:2CD5,2CD6:2CD7,2CD8:2CD9,2CDA:2CDB,2CDC:2CDD,2CDE:2CDF,2CE0:2CE1,2CE2:2CE3,2CEB:2CEC,2CED:2CEE,2CF2:2CF3,A640:A641,A642:A643,A644:A645,A646:A647,A648:A649,A64A:A64B,A64C:A64D,A64E:A64F,A650:A651,A652:A653,A654:A655,A656:A657,A658:A659,A65A:A65B,A65C:A65D,A65E:A65F,A660:A661,A662:A663,A664:A665,A666:A667,A668:A669,A66A:A66B,A66C:A66D,A680:A681,A682:A683,A684:A685,A686:A687,A688:A689,A68A:A68B,A68C:A68D,A68E:A68F,A690:A691,A692:A693,A694:A695,A696:A697,A698:A699,A69A:A69B,A722:A723,A724:A725,A726:A727,A728:A729,A72A:A72B,A72C:A72D,A72E:A72F,A732:A733,A734:A735,A736:A737,A738:A739,A73A:A73B,A73C:A73D,A73E:A73F,A740:A741,A742:A743,A744:A745,A746:A747,A748:A749,A74A:A74B,A74C:A74D,A74E:A74F,A750:A751,A752:A753,A754:A755,A756:A757,A758:A759,A75A:A75B,A75C:A75D,A75E:A75F,A760:A761,A762:A763,A764:A765,A766:A767,A768:A769,A76A:A76B,A76C:A76D,A76E:A76F,A779:A77A,A77B:A77C,A77D:1D79,A77E:A77F,A780:A781,A782:A783,A784:A785,A786:A787,A78B:A78C,A78D:265,A790:A791,A792:A793,A796:A797,A798:A799,A79A:A79B,A79C:A79D,A79E:A79F,A7A0:A7A1,A7A2:A7A3,A7A4:A7A5,A7A6:A7A7,A7A8:A7A9,A7AA:266,A7AB:25C,A7AC:261,A7AD:26C,A7AE:26A,A7B0:29E,A7B1:287,A7B2:29D,A7B3:AB53,A7B4:A7B5,A7B6:A7B7,A7B8:A7B9,A7BA:A7BB,A7BC:A7BD,A7BE:A7BF,A7C0:A7C1,A7C2:A7C3,A7C4:A794,A7C5:282,A7C6:1D8E,A7C7:A7C8,A7C9:A7CA,A7CB:264,A7CC:A7CD,A7D0:A7D1,A7D6:A7D7,A7D8:A7D9,A7DA:A7DB,A7DC:19B,A7F5:A7F6,FF21:FF41,FF22:FF42,FF23:FF43,FF24:FF44,FF25:FF45,FF26:FF46,FF27:FF47,FF28:FF48,FF29:FF49,FF2A:FF4A,FF2B:FF4B,FF2C:FF4C,FF2D:FF4D,FF2E:FF4E,FF2F:FF4F,FF30:FF50,FF31:FF51,FF32:FF52,FF33:FF53,FF34:FF54,FF35:FF55,FF36:FF56,FF37:FF57,FF38:FF58,FF39:FF59,FF3A:FF5A,10400:10428,10401:10429,10402:1042A,10403:1042B,10404:1042C,10405:1042D,10406:1042E,10407:1042F,10408:10430,10409:10431,1040A:10432,1040B:10433,1040C:10434,1040D:10435,1040E:10436,1040F:10437,10410:10438,10411:10439,10412:1043A,10413:1043B,10414:1043C,10415:1043D,10416:1043E,10417:1043F,10418:10440,10419:10441,1041A:10442,1041B:10443,1041C:10444,1041D:10445,1041E:10446,1041F:10447,10420:10448,10421:10449,10422:1044A,10423:1044B,10424:1044C,10425:1044D,10426:1044E,10427:1044F,104B0:104D8,104B1:104D9,104B2:104DA,104B3:104DB,104B4:104DC,104B5:104DD,104B6:104DE,104B7:104DF,104B8:104E0,104B9:104E1,104BA:104E2,104BB:104E3,104BC:104E4,104BD:104E5,104BE:104E6,104BF:104E7,104C0:104E8,104C1:104E9,104C2:104EA,104C3:104EB,104C4:104EC,104C5:104ED,104C6:104EE,104C7:104EF,104C8:104F0,104C9:104F1,104CA:104F2,104CB:104F3,104CC:104F4,104CD:104F5,104CE:104F6,104CF:104F7,104D0:104F8,104D1:104F9,104D2:104FA,104D3:104FB,10570:10597,10571:10598,10572:10599,10573:1059A,10574:1059B,10575:1059C,10576:1059D,10577:1059E,10578:1059F,10579:105A0,1057A:105A1,1057C:105A3,1057D:105A4,1057E:105A5,1057F:105A6,10580:105A7,10581:105A8,10582:105A9,10583:105AA,10584:105AB,10585:105AC,10586:105AD,10587:105AE,10588:105AF,10589:105B0,1058A:105B1,1058C:105B3,1058D:105B4,1058E:105B5,1058F:105B6,10590:105B7,10591:105B8,10592:105B9,10594:105BB,10595:105BC,10C80:10CC0,10C81:10CC1,10C82:10CC2,10C83:10CC3,10C84:10CC4,10C85:10CC5,10C86:10CC6,10C87:10CC7,10C88:10CC8,10C89:10CC9,10C8A:10CCA,10C8B:10CCB,10C8C:10CCC,10C8D:10CCD,10C8E:10CCE,10C8F:10CCF,10C90:10CD0,10C91:10CD1,10C92:10CD2,10C93:10CD3,10C94:10CD4,10C95:10CD5,10C96:10CD6,10C97:10CD7,10C98:10CD8,10C99:10CD9,10C9A:10CDA,10C9B:10CDB,10C9C:10CDC,10C9D:10CDD,10C9E:10CDE,10C9F:10CDF,10CA0:10CE0,10CA1:10CE1,10CA2:10CE2,10CA3:10CE3,10CA4:10CE4,10CA5:10CE5,10CA6:10CE6,10CA7:10CE7,10CA8:10CE8,10CA9:10CE9,10CAA:10CEA,10CAB:10CEB,10CAC:10CEC,10CAD:10CED,10CAE:10CEE,10CAF:10CEF,10CB0:10CF0,10CB1:10CF1,10CB2:10CF2,10D50:10D70,10D51:10D71,10D52:10D72,10D53:10D73,10D54:10D74,10D55:10D75,10D56:10D76,10D57:10D77,10D58:10D78,10D59:10D79,10D5A:10D7A,10D5B:10D7B,10D5C:10D7C,10D5D:10D7D,10D5E:10D7E,10D5F:10D7F,10D60:10D80,10D61:10D81,10D62:10D82,10D63:10D83,10D64:10D84,10D65:10D85,118A0:118C0,118A1:118C1,118A2:118C2,118A3:118C3,118A4:118C4,118A5:118C5,118A6:118C6,118A7:118C7,118A8:118C8,118A9:118C9,118AA:118CA,118AB:118CB,118AC:118CC,118AD:118CD,118AE:118CE,118AF:118CF,118B0:118D0,118B1:118D1,118B2:118D2,118B3:118D3,118B4:118D4,118B5:118D5,118B6:118D6,118B7:118D7,118B8:118D8,118B9:118D9,118BA:118DA,118BB:118DB,118BC:118DC,118BD:118DD,118BE:118DE,118BF:118DF,16E40:16E60,16E41:16E61,16E42:16E62,16E43:16E63,16E44:16E64,16E45:16E65,16E46:16E66,16E47:16E67,16E48:16E68,16E49:16E69,16E4A:16E6A,16E4B:16E6B,16E4C:16E6C,16E4D:16E6D,16E4E:16E6E,16E4F:16E6F,16E50:16E70,16E51:16E71,16E52:16E72,16E53:16E73,16E54:16E74,16E55:16E75,16E56:16E76,16E57:16E77,16E58:16E78,16E59:16E79,16E5A:16E7A,16E5B:16E7B,16E5C:16E7C,16E5D:16E7D,16E5E:16E7E,16E5F:16E7F,1E900:1E922,1E901:1E923,1E902:1E924,1E903:1E925,1E904:1E926,1E905:1E927,1E906:1E928,1E907:1E929,1E908:1E92A,1E909:1E92B,1E90A:1E92C,1E90B:1E92D,1E90C:1E92E,1E90D:1E92F,1E90E:1E930,1E90F:1E931,1E910:1E932,1E911:1E933,1E912:1E934,1E913:1E935,1E914:1E936,1E915:1E937,1E916:1E938,1E917:1E939,1E918:1E93A,1E919:1E93B,1E91A:1E93C,1E91B:1E93D,1E91C:1E93E,1E91D:1E93F,1E91E:1E940,1E91F:1E941,1E920:1E942,1E921:1E943 diff --git a/crates/unicode/tests/data/cpython3.14_predicates.txt b/crates/unicode/tests/data/cpython3.14_predicates.txt index 9339f4e4744..22f9d8d6105 100644 --- a/crates/unicode/tests/data/cpython3.14_predicates.txt +++ b/crates/unicode/tests/data/cpython3.14_predicates.txt @@ -7,3 +7,7 @@ isnumeric 30:39,B2:B3,B9:B9,BC:BE,660:669,6F0:6F9,7C0:7C9,966:96F,9E6:9EF,9F4:9F isspace 9:D,1C:20,85:85,A0:A0,1680:1680,2000:200A,2028:2029,202F:202F,205F:205F,3000:3000 isprintable 20:7E,A1:AC,AE:377,37A:37F,384:38A,38C:38C,38E:3A1,3A3:52F,531:556,559:58A,58D:58F,591:5C7,5D0:5EA,5EF:5F4,606:61B,61D:6DC,6DE:70D,710:74A,74D:7B1,7C0:7FA,7FD:82D,830:83E,840:85B,85E:85E,860:86A,870:88E,897:8E1,8E3:983,985:98C,98F:990,993:9A8,9AA:9B0,9B2:9B2,9B6:9B9,9BC:9C4,9C7:9C8,9CB:9CE,9D7:9D7,9DC:9DD,9DF:9E3,9E6:9FE,A01:A03,A05:A0A,A0F:A10,A13:A28,A2A:A30,A32:A33,A35:A36,A38:A39,A3C:A3C,A3E:A42,A47:A48,A4B:A4D,A51:A51,A59:A5C,A5E:A5E,A66:A76,A81:A83,A85:A8D,A8F:A91,A93:AA8,AAA:AB0,AB2:AB3,AB5:AB9,ABC:AC5,AC7:AC9,ACB:ACD,AD0:AD0,AE0:AE3,AE6:AF1,AF9:AFF,B01:B03,B05:B0C,B0F:B10,B13:B28,B2A:B30,B32:B33,B35:B39,B3C:B44,B47:B48,B4B:B4D,B55:B57,B5C:B5D,B5F:B63,B66:B77,B82:B83,B85:B8A,B8E:B90,B92:B95,B99:B9A,B9C:B9C,B9E:B9F,BA3:BA4,BA8:BAA,BAE:BB9,BBE:BC2,BC6:BC8,BCA:BCD,BD0:BD0,BD7:BD7,BE6:BFA,C00:C0C,C0E:C10,C12:C28,C2A:C39,C3C:C44,C46:C48,C4A:C4D,C55:C56,C58:C5A,C5D:C5D,C60:C63,C66:C6F,C77:C8C,C8E:C90,C92:CA8,CAA:CB3,CB5:CB9,CBC:CC4,CC6:CC8,CCA:CCD,CD5:CD6,CDD:CDE,CE0:CE3,CE6:CEF,CF1:CF3,D00:D0C,D0E:D10,D12:D44,D46:D48,D4A:D4F,D54:D63,D66:D7F,D81:D83,D85:D96,D9A:DB1,DB3:DBB,DBD:DBD,DC0:DC6,DCA:DCA,DCF:DD4,DD6:DD6,DD8:DDF,DE6:DEF,DF2:DF4,E01:E3A,E3F:E5B,E81:E82,E84:E84,E86:E8A,E8C:EA3,EA5:EA5,EA7:EBD,EC0:EC4,EC6:EC6,EC8:ECE,ED0:ED9,EDC:EDF,F00:F47,F49:F6C,F71:F97,F99:FBC,FBE:FCC,FCE:FDA,1000:10C5,10C7:10C7,10CD:10CD,10D0:1248,124A:124D,1250:1256,1258:1258,125A:125D,1260:1288,128A:128D,1290:12B0,12B2:12B5,12B8:12BE,12C0:12C0,12C2:12C5,12C8:12D6,12D8:1310,1312:1315,1318:135A,135D:137C,1380:1399,13A0:13F5,13F8:13FD,1400:167F,1681:169C,16A0:16F8,1700:1715,171F:1736,1740:1753,1760:176C,176E:1770,1772:1773,1780:17DD,17E0:17E9,17F0:17F9,1800:180D,180F:1819,1820:1878,1880:18AA,18B0:18F5,1900:191E,1920:192B,1930:193B,1940:1940,1944:196D,1970:1974,1980:19AB,19B0:19C9,19D0:19DA,19DE:1A1B,1A1E:1A5E,1A60:1A7C,1A7F:1A89,1A90:1A99,1AA0:1AAD,1AB0:1ACE,1B00:1B4C,1B4E:1BF3,1BFC:1C37,1C3B:1C49,1C4D:1C8A,1C90:1CBA,1CBD:1CC7,1CD0:1CFA,1D00:1F15,1F18:1F1D,1F20:1F45,1F48:1F4D,1F50:1F57,1F59:1F59,1F5B:1F5B,1F5D:1F5D,1F5F:1F7D,1F80:1FB4,1FB6:1FC4,1FC6:1FD3,1FD6:1FDB,1FDD:1FEF,1FF2:1FF4,1FF6:1FFE,2010:2027,2030:205E,2070:2071,2074:208E,2090:209C,20A0:20C0,20D0:20F0,2100:218B,2190:2429,2440:244A,2460:2B73,2B76:2B95,2B97:2CF3,2CF9:2D25,2D27:2D27,2D2D:2D2D,2D30:2D67,2D6F:2D70,2D7F:2D96,2DA0:2DA6,2DA8:2DAE,2DB0:2DB6,2DB8:2DBE,2DC0:2DC6,2DC8:2DCE,2DD0:2DD6,2DD8:2DDE,2DE0:2E5D,2E80:2E99,2E9B:2EF3,2F00:2FD5,2FF0:2FFF,3001:303F,3041:3096,3099:30FF,3105:312F,3131:318E,3190:31E5,31EF:321E,3220:A48C,A490:A4C6,A4D0:A62B,A640:A6F7,A700:A7CD,A7D0:A7D1,A7D3:A7D3,A7D5:A7DC,A7F2:A82C,A830:A839,A840:A877,A880:A8C5,A8CE:A8D9,A8E0:A953,A95F:A97C,A980:A9CD,A9CF:A9D9,A9DE:A9FE,AA00:AA36,AA40:AA4D,AA50:AA59,AA5C:AAC2,AADB:AAF6,AB01:AB06,AB09:AB0E,AB11:AB16,AB20:AB26,AB28:AB2E,AB30:AB6B,AB70:ABED,ABF0:ABF9,AC00:D7A3,D7B0:D7C6,D7CB:D7FB,F900:FA6D,FA70:FAD9,FB00:FB06,FB13:FB17,FB1D:FB36,FB38:FB3C,FB3E:FB3E,FB40:FB41,FB43:FB44,FB46:FBC2,FBD3:FD8F,FD92:FDC7,FDCF:FDCF,FDF0:FE19,FE20:FE52,FE54:FE66,FE68:FE6B,FE70:FE74,FE76:FEFC,FF01:FFBE,FFC2:FFC7,FFCA:FFCF,FFD2:FFD7,FFDA:FFDC,FFE0:FFE6,FFE8:FFEE,FFFC:FFFD,10000:1000B,1000D:10026,10028:1003A,1003C:1003D,1003F:1004D,10050:1005D,10080:100FA,10100:10102,10107:10133,10137:1018E,10190:1019C,101A0:101A0,101D0:101FD,10280:1029C,102A0:102D0,102E0:102FB,10300:10323,1032D:1034A,10350:1037A,10380:1039D,1039F:103C3,103C8:103D5,10400:1049D,104A0:104A9,104B0:104D3,104D8:104FB,10500:10527,10530:10563,1056F:1057A,1057C:1058A,1058C:10592,10594:10595,10597:105A1,105A3:105B1,105B3:105B9,105BB:105BC,105C0:105F3,10600:10736,10740:10755,10760:10767,10780:10785,10787:107B0,107B2:107BA,10800:10805,10808:10808,1080A:10835,10837:10838,1083C:1083C,1083F:10855,10857:1089E,108A7:108AF,108E0:108F2,108F4:108F5,108FB:1091B,1091F:10939,1093F:1093F,10980:109B7,109BC:109CF,109D2:10A03,10A05:10A06,10A0C:10A13,10A15:10A17,10A19:10A35,10A38:10A3A,10A3F:10A48,10A50:10A58,10A60:10A9F,10AC0:10AE6,10AEB:10AF6,10B00:10B35,10B39:10B55,10B58:10B72,10B78:10B91,10B99:10B9C,10BA9:10BAF,10C00:10C48,10C80:10CB2,10CC0:10CF2,10CFA:10D27,10D30:10D39,10D40:10D65,10D69:10D85,10D8E:10D8F,10E60:10E7E,10E80:10EA9,10EAB:10EAD,10EB0:10EB1,10EC2:10EC4,10EFC:10F27,10F30:10F59,10F70:10F89,10FB0:10FCB,10FE0:10FF6,11000:1104D,11052:11075,1107F:110BC,110BE:110C2,110D0:110E8,110F0:110F9,11100:11134,11136:11147,11150:11176,11180:111DF,111E1:111F4,11200:11211,11213:11241,11280:11286,11288:11288,1128A:1128D,1128F:1129D,1129F:112A9,112B0:112EA,112F0:112F9,11300:11303,11305:1130C,1130F:11310,11313:11328,1132A:11330,11332:11333,11335:11339,1133B:11344,11347:11348,1134B:1134D,11350:11350,11357:11357,1135D:11363,11366:1136C,11370:11374,11380:11389,1138B:1138B,1138E:1138E,11390:113B5,113B7:113C0,113C2:113C2,113C5:113C5,113C7:113CA,113CC:113D5,113D7:113D8,113E1:113E2,11400:1145B,1145D:11461,11480:114C7,114D0:114D9,11580:115B5,115B8:115DD,11600:11644,11650:11659,11660:1166C,11680:116B9,116C0:116C9,116D0:116E3,11700:1171A,1171D:1172B,11730:11746,11800:1183B,118A0:118F2,118FF:11906,11909:11909,1190C:11913,11915:11916,11918:11935,11937:11938,1193B:11946,11950:11959,119A0:119A7,119AA:119D7,119DA:119E4,11A00:11A47,11A50:11AA2,11AB0:11AF8,11B00:11B09,11BC0:11BE1,11BF0:11BF9,11C00:11C08,11C0A:11C36,11C38:11C45,11C50:11C6C,11C70:11C8F,11C92:11CA7,11CA9:11CB6,11D00:11D06,11D08:11D09,11D0B:11D36,11D3A:11D3A,11D3C:11D3D,11D3F:11D47,11D50:11D59,11D60:11D65,11D67:11D68,11D6A:11D8E,11D90:11D91,11D93:11D98,11DA0:11DA9,11EE0:11EF8,11F00:11F10,11F12:11F3A,11F3E:11F5A,11FB0:11FB0,11FC0:11FF1,11FFF:12399,12400:1246E,12470:12474,12480:12543,12F90:12FF2,13000:1342F,13440:13455,13460:143FA,14400:14646,16100:16139,16800:16A38,16A40:16A5E,16A60:16A69,16A6E:16ABE,16AC0:16AC9,16AD0:16AED,16AF0:16AF5,16B00:16B45,16B50:16B59,16B5B:16B61,16B63:16B77,16B7D:16B8F,16D40:16D79,16E40:16E9A,16F00:16F4A,16F4F:16F87,16F8F:16F9F,16FE0:16FE4,16FF0:16FF1,17000:187F7,18800:18CD5,18CFF:18D08,1AFF0:1AFF3,1AFF5:1AFFB,1AFFD:1AFFE,1B000:1B122,1B132:1B132,1B150:1B152,1B155:1B155,1B164:1B167,1B170:1B2FB,1BC00:1BC6A,1BC70:1BC7C,1BC80:1BC88,1BC90:1BC99,1BC9C:1BC9F,1CC00:1CCF9,1CD00:1CEB3,1CF00:1CF2D,1CF30:1CF46,1CF50:1CFC3,1D000:1D0F5,1D100:1D126,1D129:1D172,1D17B:1D1EA,1D200:1D245,1D2C0:1D2D3,1D2E0:1D2F3,1D300:1D356,1D360:1D378,1D400:1D454,1D456:1D49C,1D49E:1D49F,1D4A2:1D4A2,1D4A5:1D4A6,1D4A9:1D4AC,1D4AE:1D4B9,1D4BB:1D4BB,1D4BD:1D4C3,1D4C5:1D505,1D507:1D50A,1D50D:1D514,1D516:1D51C,1D51E:1D539,1D53B:1D53E,1D540:1D544,1D546:1D546,1D54A:1D550,1D552:1D6A5,1D6A8:1D7CB,1D7CE:1DA8B,1DA9B:1DA9F,1DAA1:1DAAF,1DF00:1DF1E,1DF25:1DF2A,1E000:1E006,1E008:1E018,1E01B:1E021,1E023:1E024,1E026:1E02A,1E030:1E06D,1E08F:1E08F,1E100:1E12C,1E130:1E13D,1E140:1E149,1E14E:1E14F,1E290:1E2AE,1E2C0:1E2F9,1E2FF:1E2FF,1E4D0:1E4F9,1E5D0:1E5FA,1E5FF:1E5FF,1E7E0:1E7E6,1E7E8:1E7EB,1E7ED:1E7EE,1E7F0:1E7FE,1E800:1E8C4,1E8C7:1E8D6,1E900:1E94B,1E950:1E959,1E95E:1E95F,1EC71:1ECB4,1ED01:1ED3D,1EE00:1EE03,1EE05:1EE1F,1EE21:1EE22,1EE24:1EE24,1EE27:1EE27,1EE29:1EE32,1EE34:1EE37,1EE39:1EE39,1EE3B:1EE3B,1EE42:1EE42,1EE47:1EE47,1EE49:1EE49,1EE4B:1EE4B,1EE4D:1EE4F,1EE51:1EE52,1EE54:1EE54,1EE57:1EE57,1EE59:1EE59,1EE5B:1EE5B,1EE5D:1EE5D,1EE5F:1EE5F,1EE61:1EE62,1EE64:1EE64,1EE67:1EE6A,1EE6C:1EE72,1EE74:1EE77,1EE79:1EE7C,1EE7E:1EE7E,1EE80:1EE89,1EE8B:1EE9B,1EEA1:1EEA3,1EEA5:1EEA9,1EEAB:1EEBB,1EEF0:1EEF1,1F000:1F02B,1F030:1F093,1F0A0:1F0AE,1F0B1:1F0BF,1F0C1:1F0CF,1F0D1:1F0F5,1F100:1F1AD,1F1E6:1F202,1F210:1F23B,1F240:1F248,1F250:1F251,1F260:1F265,1F300:1F6D7,1F6DC:1F6EC,1F6F0:1F6FC,1F700:1F776,1F77B:1F7D9,1F7E0:1F7EB,1F7F0:1F7F0,1F800:1F80B,1F810:1F847,1F850:1F859,1F860:1F887,1F890:1F8AD,1F8B0:1F8BB,1F8C0:1F8C1,1F900:1FA53,1FA60:1FA6D,1FA70:1FA7C,1FA80:1FA89,1FA8F:1FAC6,1FACE:1FADC,1FADF:1FAE9,1FAF0:1FAF8,1FB00:1FB92,1FB94:1FBF9,20000:2A6DF,2A700:2B739,2B740:2B81D,2B820:2CEA1,2CEB0:2EBE0,2EBF0:2EE5D,2F800:2FA1D,30000:3134A,31350:323AF,E0100:E01EF isidentifier 41:5A,5F:5F,61:7A,AA:AA,B5:B5,BA:BA,C0:D6,D8:F6,F8:2C1,2C6:2D1,2E0:2E4,2EC:2EC,2EE:2EE,370:374,376:377,37B:37D,37F:37F,386:386,388:38A,38C:38C,38E:3A1,3A3:3F5,3F7:481,48A:52F,531:556,559:559,560:588,5D0:5EA,5EF:5F2,620:64A,66E:66F,671:6D3,6D5:6D5,6E5:6E6,6EE:6EF,6FA:6FC,6FF:6FF,710:710,712:72F,74D:7A5,7B1:7B1,7CA:7EA,7F4:7F5,7FA:7FA,800:815,81A:81A,824:824,828:828,840:858,860:86A,870:887,889:88E,8A0:8C9,904:939,93D:93D,950:950,958:961,971:980,985:98C,98F:990,993:9A8,9AA:9B0,9B2:9B2,9B6:9B9,9BD:9BD,9CE:9CE,9DC:9DD,9DF:9E1,9F0:9F1,9FC:9FC,A05:A0A,A0F:A10,A13:A28,A2A:A30,A32:A33,A35:A36,A38:A39,A59:A5C,A5E:A5E,A72:A74,A85:A8D,A8F:A91,A93:AA8,AAA:AB0,AB2:AB3,AB5:AB9,ABD:ABD,AD0:AD0,AE0:AE1,AF9:AF9,B05:B0C,B0F:B10,B13:B28,B2A:B30,B32:B33,B35:B39,B3D:B3D,B5C:B5D,B5F:B61,B71:B71,B83:B83,B85:B8A,B8E:B90,B92:B95,B99:B9A,B9C:B9C,B9E:B9F,BA3:BA4,BA8:BAA,BAE:BB9,BD0:BD0,C05:C0C,C0E:C10,C12:C28,C2A:C39,C3D:C3D,C58:C5A,C5D:C5D,C60:C61,C80:C80,C85:C8C,C8E:C90,C92:CA8,CAA:CB3,CB5:CB9,CBD:CBD,CDD:CDE,CE0:CE1,CF1:CF2,D04:D0C,D0E:D10,D12:D3A,D3D:D3D,D4E:D4E,D54:D56,D5F:D61,D7A:D7F,D85:D96,D9A:DB1,DB3:DBB,DBD:DBD,DC0:DC6,E01:E30,E32:E32,E40:E46,E81:E82,E84:E84,E86:E8A,E8C:EA3,EA5:EA5,EA7:EB0,EB2:EB2,EBD:EBD,EC0:EC4,EC6:EC6,EDC:EDF,F00:F00,F40:F47,F49:F6C,F88:F8C,1000:102A,103F:103F,1050:1055,105A:105D,1061:1061,1065:1066,106E:1070,1075:1081,108E:108E,10A0:10C5,10C7:10C7,10CD:10CD,10D0:10FA,10FC:1248,124A:124D,1250:1256,1258:1258,125A:125D,1260:1288,128A:128D,1290:12B0,12B2:12B5,12B8:12BE,12C0:12C0,12C2:12C5,12C8:12D6,12D8:1310,1312:1315,1318:135A,1380:138F,13A0:13F5,13F8:13FD,1401:166C,166F:167F,1681:169A,16A0:16EA,16EE:16F8,1700:1711,171F:1731,1740:1751,1760:176C,176E:1770,1780:17B3,17D7:17D7,17DC:17DC,1820:1878,1880:18A8,18AA:18AA,18B0:18F5,1900:191E,1950:196D,1970:1974,1980:19AB,19B0:19C9,1A00:1A16,1A20:1A54,1AA7:1AA7,1B05:1B33,1B45:1B4C,1B83:1BA0,1BAE:1BAF,1BBA:1BE5,1C00:1C23,1C4D:1C4F,1C5A:1C7D,1C80:1C8A,1C90:1CBA,1CBD:1CBF,1CE9:1CEC,1CEE:1CF3,1CF5:1CF6,1CFA:1CFA,1D00:1DBF,1E00:1F15,1F18:1F1D,1F20:1F45,1F48:1F4D,1F50:1F57,1F59:1F59,1F5B:1F5B,1F5D:1F5D,1F5F:1F7D,1F80:1FB4,1FB6:1FBC,1FBE:1FBE,1FC2:1FC4,1FC6:1FCC,1FD0:1FD3,1FD6:1FDB,1FE0:1FEC,1FF2:1FF4,1FF6:1FFC,2071:2071,207F:207F,2090:209C,2102:2102,2107:2107,210A:2113,2115:2115,2118:211D,2124:2124,2126:2126,2128:2128,212A:2139,213C:213F,2145:2149,214E:214E,2160:2188,2C00:2CE4,2CEB:2CEE,2CF2:2CF3,2D00:2D25,2D27:2D27,2D2D:2D2D,2D30:2D67,2D6F:2D6F,2D80:2D96,2DA0:2DA6,2DA8:2DAE,2DB0:2DB6,2DB8:2DBE,2DC0:2DC6,2DC8:2DCE,2DD0:2DD6,2DD8:2DDE,3005:3007,3021:3029,3031:3035,3038:303C,3041:3096,309D:309F,30A1:30FA,30FC:30FF,3105:312F,3131:318E,31A0:31BF,31F0:31FF,3400:4DBF,4E00:A48C,A4D0:A4FD,A500:A60C,A610:A61F,A62A:A62B,A640:A66E,A67F:A69D,A6A0:A6EF,A717:A71F,A722:A788,A78B:A7CD,A7D0:A7D1,A7D3:A7D3,A7D5:A7DC,A7F2:A801,A803:A805,A807:A80A,A80C:A822,A840:A873,A882:A8B3,A8F2:A8F7,A8FB:A8FB,A8FD:A8FE,A90A:A925,A930:A946,A960:A97C,A984:A9B2,A9CF:A9CF,A9E0:A9E4,A9E6:A9EF,A9FA:A9FE,AA00:AA28,AA40:AA42,AA44:AA4B,AA60:AA76,AA7A:AA7A,AA7E:AAAF,AAB1:AAB1,AAB5:AAB6,AAB9:AABD,AAC0:AAC0,AAC2:AAC2,AADB:AADD,AAE0:AAEA,AAF2:AAF4,AB01:AB06,AB09:AB0E,AB11:AB16,AB20:AB26,AB28:AB2E,AB30:AB5A,AB5C:AB69,AB70:ABE2,AC00:D7A3,D7B0:D7C6,D7CB:D7FB,F900:FA6D,FA70:FAD9,FB00:FB06,FB13:FB17,FB1D:FB1D,FB1F:FB28,FB2A:FB36,FB38:FB3C,FB3E:FB3E,FB40:FB41,FB43:FB44,FB46:FBB1,FBD3:FC5D,FC64:FD3D,FD50:FD8F,FD92:FDC7,FDF0:FDF9,FE71:FE71,FE73:FE73,FE77:FE77,FE79:FE79,FE7B:FE7B,FE7D:FE7D,FE7F:FEFC,FF21:FF3A,FF41:FF5A,FF66:FF9D,FFA0:FFBE,FFC2:FFC7,FFCA:FFCF,FFD2:FFD7,FFDA:FFDC,10000:1000B,1000D:10026,10028:1003A,1003C:1003D,1003F:1004D,10050:1005D,10080:100FA,10140:10174,10280:1029C,102A0:102D0,10300:1031F,1032D:1034A,10350:10375,10380:1039D,103A0:103C3,103C8:103CF,103D1:103D5,10400:1049D,104B0:104D3,104D8:104FB,10500:10527,10530:10563,10570:1057A,1057C:1058A,1058C:10592,10594:10595,10597:105A1,105A3:105B1,105B3:105B9,105BB:105BC,105C0:105F3,10600:10736,10740:10755,10760:10767,10780:10785,10787:107B0,107B2:107BA,10800:10805,10808:10808,1080A:10835,10837:10838,1083C:1083C,1083F:10855,10860:10876,10880:1089E,108E0:108F2,108F4:108F5,10900:10915,10920:10939,10980:109B7,109BE:109BF,10A00:10A00,10A10:10A13,10A15:10A17,10A19:10A35,10A60:10A7C,10A80:10A9C,10AC0:10AC7,10AC9:10AE4,10B00:10B35,10B40:10B55,10B60:10B72,10B80:10B91,10C00:10C48,10C80:10CB2,10CC0:10CF2,10D00:10D23,10D4A:10D65,10D6F:10D85,10E80:10EA9,10EB0:10EB1,10EC2:10EC4,10F00:10F1C,10F27:10F27,10F30:10F45,10F70:10F81,10FB0:10FC4,10FE0:10FF6,11003:11037,11071:11072,11075:11075,11083:110AF,110D0:110E8,11103:11126,11144:11144,11147:11147,11150:11172,11176:11176,11183:111B2,111C1:111C4,111DA:111DA,111DC:111DC,11200:11211,11213:1122B,1123F:11240,11280:11286,11288:11288,1128A:1128D,1128F:1129D,1129F:112A8,112B0:112DE,11305:1130C,1130F:11310,11313:11328,1132A:11330,11332:11333,11335:11339,1133D:1133D,11350:11350,1135D:11361,11380:11389,1138B:1138B,1138E:1138E,11390:113B5,113B7:113B7,113D1:113D1,113D3:113D3,11400:11434,11447:1144A,1145F:11461,11480:114AF,114C4:114C5,114C7:114C7,11580:115AE,115D8:115DB,11600:1162F,11644:11644,11680:116AA,116B8:116B8,11700:1171A,11740:11746,11800:1182B,118A0:118DF,118FF:11906,11909:11909,1190C:11913,11915:11916,11918:1192F,1193F:1193F,11941:11941,119A0:119A7,119AA:119D0,119E1:119E1,119E3:119E3,11A00:11A00,11A0B:11A32,11A3A:11A3A,11A50:11A50,11A5C:11A89,11A9D:11A9D,11AB0:11AF8,11BC0:11BE0,11C00:11C08,11C0A:11C2E,11C40:11C40,11C72:11C8F,11D00:11D06,11D08:11D09,11D0B:11D30,11D46:11D46,11D60:11D65,11D67:11D68,11D6A:11D89,11D98:11D98,11EE0:11EF2,11F02:11F02,11F04:11F10,11F12:11F33,11FB0:11FB0,12000:12399,12400:1246E,12480:12543,12F90:12FF0,13000:1342F,13441:13446,13460:143FA,14400:14646,16100:1611D,16800:16A38,16A40:16A5E,16A70:16ABE,16AD0:16AED,16B00:16B2F,16B40:16B43,16B63:16B77,16B7D:16B8F,16D40:16D6C,16E40:16E7F,16F00:16F4A,16F50:16F50,16F93:16F9F,16FE0:16FE1,16FE3:16FE3,17000:187F7,18800:18CD5,18CFF:18D08,1AFF0:1AFF3,1AFF5:1AFFB,1AFFD:1AFFE,1B000:1B122,1B132:1B132,1B150:1B152,1B155:1B155,1B164:1B167,1B170:1B2FB,1BC00:1BC6A,1BC70:1BC7C,1BC80:1BC88,1BC90:1BC99,1D400:1D454,1D456:1D49C,1D49E:1D49F,1D4A2:1D4A2,1D4A5:1D4A6,1D4A9:1D4AC,1D4AE:1D4B9,1D4BB:1D4BB,1D4BD:1D4C3,1D4C5:1D505,1D507:1D50A,1D50D:1D514,1D516:1D51C,1D51E:1D539,1D53B:1D53E,1D540:1D544,1D546:1D546,1D54A:1D550,1D552:1D6A5,1D6A8:1D6C0,1D6C2:1D6DA,1D6DC:1D6FA,1D6FC:1D714,1D716:1D734,1D736:1D74E,1D750:1D76E,1D770:1D788,1D78A:1D7A8,1D7AA:1D7C2,1D7C4:1D7CB,1DF00:1DF1E,1DF25:1DF2A,1E030:1E06D,1E100:1E12C,1E137:1E13D,1E14E:1E14E,1E290:1E2AD,1E2C0:1E2EB,1E4D0:1E4EB,1E5D0:1E5ED,1E5F0:1E5F0,1E7E0:1E7E6,1E7E8:1E7EB,1E7ED:1E7EE,1E7F0:1E7FE,1E800:1E8C4,1E900:1E943,1E94B:1E94B,1EE00:1EE03,1EE05:1EE1F,1EE21:1EE22,1EE24:1EE24,1EE27:1EE27,1EE29:1EE32,1EE34:1EE37,1EE39:1EE39,1EE3B:1EE3B,1EE42:1EE42,1EE47:1EE47,1EE49:1EE49,1EE4B:1EE4B,1EE4D:1EE4F,1EE51:1EE52,1EE54:1EE54,1EE57:1EE57,1EE59:1EE59,1EE5B:1EE5B,1EE5D:1EE5D,1EE5F:1EE5F,1EE61:1EE62,1EE64:1EE64,1EE67:1EE6A,1EE6C:1EE72,1EE74:1EE77,1EE79:1EE7C,1EE7E:1EE7E,1EE80:1EE89,1EE8B:1EE9B,1EEA1:1EEA3,1EEA5:1EEA9,1EEAB:1EEBB,20000:2A6DF,2A700:2B739,2B740:2B81D,2B820:2CEA1,2CEB0:2EBE0,2EBF0:2EE5D,2F800:2FA1D,30000:3134A,31350:323AF +is_lowercase 61:7A,AA:AA,B5:B5,BA:BA,DF:F6,F8:FF,101:101,103:103,105:105,107:107,109:109,10B:10B,10D:10D,10F:10F,111:111,113:113,115:115,117:117,119:119,11B:11B,11D:11D,11F:11F,121:121,123:123,125:125,127:127,129:129,12B:12B,12D:12D,12F:12F,131:131,133:133,135:135,137:138,13A:13A,13C:13C,13E:13E,140:140,142:142,144:144,146:146,148:149,14B:14B,14D:14D,14F:14F,151:151,153:153,155:155,157:157,159:159,15B:15B,15D:15D,15F:15F,161:161,163:163,165:165,167:167,169:169,16B:16B,16D:16D,16F:16F,171:171,173:173,175:175,177:177,17A:17A,17C:17C,17E:180,183:183,185:185,188:188,18C:18D,192:192,195:195,199:19B,19E:19E,1A1:1A1,1A3:1A3,1A5:1A5,1A8:1A8,1AA:1AB,1AD:1AD,1B0:1B0,1B4:1B4,1B6:1B6,1B9:1BA,1BD:1BF,1C6:1C6,1C9:1C9,1CC:1CC,1CE:1CE,1D0:1D0,1D2:1D2,1D4:1D4,1D6:1D6,1D8:1D8,1DA:1DA,1DC:1DD,1DF:1DF,1E1:1E1,1E3:1E3,1E5:1E5,1E7:1E7,1E9:1E9,1EB:1EB,1ED:1ED,1EF:1F0,1F3:1F3,1F5:1F5,1F9:1F9,1FB:1FB,1FD:1FD,1FF:1FF,201:201,203:203,205:205,207:207,209:209,20B:20B,20D:20D,20F:20F,211:211,213:213,215:215,217:217,219:219,21B:21B,21D:21D,21F:21F,221:221,223:223,225:225,227:227,229:229,22B:22B,22D:22D,22F:22F,231:231,233:239,23C:23C,23F:240,242:242,247:247,249:249,24B:24B,24D:24D,24F:293,295:2B8,2C0:2C1,2E0:2E4,345:345,371:371,373:373,377:377,37A:37D,390:390,3AC:3CE,3D0:3D1,3D5:3D7,3D9:3D9,3DB:3DB,3DD:3DD,3DF:3DF,3E1:3E1,3E3:3E3,3E5:3E5,3E7:3E7,3E9:3E9,3EB:3EB,3ED:3ED,3EF:3F3,3F5:3F5,3F8:3F8,3FB:3FC,430:45F,461:461,463:463,465:465,467:467,469:469,46B:46B,46D:46D,46F:46F,471:471,473:473,475:475,477:477,479:479,47B:47B,47D:47D,47F:47F,481:481,48B:48B,48D:48D,48F:48F,491:491,493:493,495:495,497:497,499:499,49B:49B,49D:49D,49F:49F,4A1:4A1,4A3:4A3,4A5:4A5,4A7:4A7,4A9:4A9,4AB:4AB,4AD:4AD,4AF:4AF,4B1:4B1,4B3:4B3,4B5:4B5,4B7:4B7,4B9:4B9,4BB:4BB,4BD:4BD,4BF:4BF,4C2:4C2,4C4:4C4,4C6:4C6,4C8:4C8,4CA:4CA,4CC:4CC,4CE:4CF,4D1:4D1,4D3:4D3,4D5:4D5,4D7:4D7,4D9:4D9,4DB:4DB,4DD:4DD,4DF:4DF,4E1:4E1,4E3:4E3,4E5:4E5,4E7:4E7,4E9:4E9,4EB:4EB,4ED:4ED,4EF:4EF,4F1:4F1,4F3:4F3,4F5:4F5,4F7:4F7,4F9:4F9,4FB:4FB,4FD:4FD,4FF:4FF,501:501,503:503,505:505,507:507,509:509,50B:50B,50D:50D,50F:50F,511:511,513:513,515:515,517:517,519:519,51B:51B,51D:51D,51F:51F,521:521,523:523,525:525,527:527,529:529,52B:52B,52D:52D,52F:52F,560:588,10D0:10FA,10FC:10FF,13F8:13FD,1C80:1C88,1C8A:1C8A,1D00:1DBF,1E01:1E01,1E03:1E03,1E05:1E05,1E07:1E07,1E09:1E09,1E0B:1E0B,1E0D:1E0D,1E0F:1E0F,1E11:1E11,1E13:1E13,1E15:1E15,1E17:1E17,1E19:1E19,1E1B:1E1B,1E1D:1E1D,1E1F:1E1F,1E21:1E21,1E23:1E23,1E25:1E25,1E27:1E27,1E29:1E29,1E2B:1E2B,1E2D:1E2D,1E2F:1E2F,1E31:1E31,1E33:1E33,1E35:1E35,1E37:1E37,1E39:1E39,1E3B:1E3B,1E3D:1E3D,1E3F:1E3F,1E41:1E41,1E43:1E43,1E45:1E45,1E47:1E47,1E49:1E49,1E4B:1E4B,1E4D:1E4D,1E4F:1E4F,1E51:1E51,1E53:1E53,1E55:1E55,1E57:1E57,1E59:1E59,1E5B:1E5B,1E5D:1E5D,1E5F:1E5F,1E61:1E61,1E63:1E63,1E65:1E65,1E67:1E67,1E69:1E69,1E6B:1E6B,1E6D:1E6D,1E6F:1E6F,1E71:1E71,1E73:1E73,1E75:1E75,1E77:1E77,1E79:1E79,1E7B:1E7B,1E7D:1E7D,1E7F:1E7F,1E81:1E81,1E83:1E83,1E85:1E85,1E87:1E87,1E89:1E89,1E8B:1E8B,1E8D:1E8D,1E8F:1E8F,1E91:1E91,1E93:1E93,1E95:1E9D,1E9F:1E9F,1EA1:1EA1,1EA3:1EA3,1EA5:1EA5,1EA7:1EA7,1EA9:1EA9,1EAB:1EAB,1EAD:1EAD,1EAF:1EAF,1EB1:1EB1,1EB3:1EB3,1EB5:1EB5,1EB7:1EB7,1EB9:1EB9,1EBB:1EBB,1EBD:1EBD,1EBF:1EBF,1EC1:1EC1,1EC3:1EC3,1EC5:1EC5,1EC7:1EC7,1EC9:1EC9,1ECB:1ECB,1ECD:1ECD,1ECF:1ECF,1ED1:1ED1,1ED3:1ED3,1ED5:1ED5,1ED7:1ED7,1ED9:1ED9,1EDB:1EDB,1EDD:1EDD,1EDF:1EDF,1EE1:1EE1,1EE3:1EE3,1EE5:1EE5,1EE7:1EE7,1EE9:1EE9,1EEB:1EEB,1EED:1EED,1EEF:1EEF,1EF1:1EF1,1EF3:1EF3,1EF5:1EF5,1EF7:1EF7,1EF9:1EF9,1EFB:1EFB,1EFD:1EFD,1EFF:1F07,1F10:1F15,1F20:1F27,1F30:1F37,1F40:1F45,1F50:1F57,1F60:1F67,1F70:1F7D,1F80:1F87,1F90:1F97,1FA0:1FA7,1FB0:1FB4,1FB6:1FB7,1FBE:1FBE,1FC2:1FC4,1FC6:1FC7,1FD0:1FD3,1FD6:1FD7,1FE0:1FE7,1FF2:1FF4,1FF6:1FF7,2071:2071,207F:207F,2090:209C,210A:210A,210E:210F,2113:2113,212F:212F,2134:2134,2139:2139,213C:213D,2146:2149,214E:214E,2170:217F,2184:2184,24D0:24E9,2C30:2C5F,2C61:2C61,2C65:2C66,2C68:2C68,2C6A:2C6A,2C6C:2C6C,2C71:2C71,2C73:2C74,2C76:2C7D,2C81:2C81,2C83:2C83,2C85:2C85,2C87:2C87,2C89:2C89,2C8B:2C8B,2C8D:2C8D,2C8F:2C8F,2C91:2C91,2C93:2C93,2C95:2C95,2C97:2C97,2C99:2C99,2C9B:2C9B,2C9D:2C9D,2C9F:2C9F,2CA1:2CA1,2CA3:2CA3,2CA5:2CA5,2CA7:2CA7,2CA9:2CA9,2CAB:2CAB,2CAD:2CAD,2CAF:2CAF,2CB1:2CB1,2CB3:2CB3,2CB5:2CB5,2CB7:2CB7,2CB9:2CB9,2CBB:2CBB,2CBD:2CBD,2CBF:2CBF,2CC1:2CC1,2CC3:2CC3,2CC5:2CC5,2CC7:2CC7,2CC9:2CC9,2CCB:2CCB,2CCD:2CCD,2CCF:2CCF,2CD1:2CD1,2CD3:2CD3,2CD5:2CD5,2CD7:2CD7,2CD9:2CD9,2CDB:2CDB,2CDD:2CDD,2CDF:2CDF,2CE1:2CE1,2CE3:2CE4,2CEC:2CEC,2CEE:2CEE,2CF3:2CF3,2D00:2D25,2D27:2D27,2D2D:2D2D,A641:A641,A643:A643,A645:A645,A647:A647,A649:A649,A64B:A64B,A64D:A64D,A64F:A64F,A651:A651,A653:A653,A655:A655,A657:A657,A659:A659,A65B:A65B,A65D:A65D,A65F:A65F,A661:A661,A663:A663,A665:A665,A667:A667,A669:A669,A66B:A66B,A66D:A66D,A681:A681,A683:A683,A685:A685,A687:A687,A689:A689,A68B:A68B,A68D:A68D,A68F:A68F,A691:A691,A693:A693,A695:A695,A697:A697,A699:A699,A69B:A69D,A723:A723,A725:A725,A727:A727,A729:A729,A72B:A72B,A72D:A72D,A72F:A731,A733:A733,A735:A735,A737:A737,A739:A739,A73B:A73B,A73D:A73D,A73F:A73F,A741:A741,A743:A743,A745:A745,A747:A747,A749:A749,A74B:A74B,A74D:A74D,A74F:A74F,A751:A751,A753:A753,A755:A755,A757:A757,A759:A759,A75B:A75B,A75D:A75D,A75F:A75F,A761:A761,A763:A763,A765:A765,A767:A767,A769:A769,A76B:A76B,A76D:A76D,A76F:A778,A77A:A77A,A77C:A77C,A77F:A77F,A781:A781,A783:A783,A785:A785,A787:A787,A78C:A78C,A78E:A78E,A791:A791,A793:A795,A797:A797,A799:A799,A79B:A79B,A79D:A79D,A79F:A79F,A7A1:A7A1,A7A3:A7A3,A7A5:A7A5,A7A7:A7A7,A7A9:A7A9,A7AF:A7AF,A7B5:A7B5,A7B7:A7B7,A7B9:A7B9,A7BB:A7BB,A7BD:A7BD,A7BF:A7BF,A7C1:A7C1,A7C3:A7C3,A7C8:A7C8,A7CA:A7CA,A7CD:A7CD,A7D1:A7D1,A7D3:A7D3,A7D5:A7D5,A7D7:A7D7,A7D9:A7D9,A7DB:A7DB,A7F2:A7F4,A7F6:A7F6,A7F8:A7FA,AB30:AB5A,AB5C:AB69,AB70:ABBF,FB00:FB06,FB13:FB17,FF41:FF5A,10428:1044F,104D8:104FB,10597:105A1,105A3:105B1,105B3:105B9,105BB:105BC,10780:10780,10783:10785,10787:107B0,107B2:107BA,10CC0:10CF2,10D70:10D85,118C0:118DF,16E60:16E7F,1D41A:1D433,1D44E:1D454,1D456:1D467,1D482:1D49B,1D4B6:1D4B9,1D4BB:1D4BB,1D4BD:1D4C3,1D4C5:1D4CF,1D4EA:1D503,1D51E:1D537,1D552:1D56B,1D586:1D59F,1D5BA:1D5D3,1D5EE:1D607,1D622:1D63B,1D656:1D66F,1D68A:1D6A5,1D6C2:1D6DA,1D6DC:1D6E1,1D6FC:1D714,1D716:1D71B,1D736:1D74E,1D750:1D755,1D770:1D788,1D78A:1D78F,1D7AA:1D7C2,1D7C4:1D7C9,1D7CB:1D7CB,1DF00:1DF09,1DF0B:1DF1E,1DF25:1DF2A,1E030:1E06D,1E922:1E943 +is_uppercase 41:5A,C0:D6,D8:DE,100:100,102:102,104:104,106:106,108:108,10A:10A,10C:10C,10E:10E,110:110,112:112,114:114,116:116,118:118,11A:11A,11C:11C,11E:11E,120:120,122:122,124:124,126:126,128:128,12A:12A,12C:12C,12E:12E,130:130,132:132,134:134,136:136,139:139,13B:13B,13D:13D,13F:13F,141:141,143:143,145:145,147:147,14A:14A,14C:14C,14E:14E,150:150,152:152,154:154,156:156,158:158,15A:15A,15C:15C,15E:15E,160:160,162:162,164:164,166:166,168:168,16A:16A,16C:16C,16E:16E,170:170,172:172,174:174,176:176,178:179,17B:17B,17D:17D,181:182,184:184,186:187,189:18B,18E:191,193:194,196:198,19C:19D,19F:1A0,1A2:1A2,1A4:1A4,1A6:1A7,1A9:1A9,1AC:1AC,1AE:1AF,1B1:1B3,1B5:1B5,1B7:1B8,1BC:1BC,1C4:1C4,1C7:1C7,1CA:1CA,1CD:1CD,1CF:1CF,1D1:1D1,1D3:1D3,1D5:1D5,1D7:1D7,1D9:1D9,1DB:1DB,1DE:1DE,1E0:1E0,1E2:1E2,1E4:1E4,1E6:1E6,1E8:1E8,1EA:1EA,1EC:1EC,1EE:1EE,1F1:1F1,1F4:1F4,1F6:1F8,1FA:1FA,1FC:1FC,1FE:1FE,200:200,202:202,204:204,206:206,208:208,20A:20A,20C:20C,20E:20E,210:210,212:212,214:214,216:216,218:218,21A:21A,21C:21C,21E:21E,220:220,222:222,224:224,226:226,228:228,22A:22A,22C:22C,22E:22E,230:230,232:232,23A:23B,23D:23E,241:241,243:246,248:248,24A:24A,24C:24C,24E:24E,370:370,372:372,376:376,37F:37F,386:386,388:38A,38C:38C,38E:38F,391:3A1,3A3:3AB,3CF:3CF,3D2:3D4,3D8:3D8,3DA:3DA,3DC:3DC,3DE:3DE,3E0:3E0,3E2:3E2,3E4:3E4,3E6:3E6,3E8:3E8,3EA:3EA,3EC:3EC,3EE:3EE,3F4:3F4,3F7:3F7,3F9:3FA,3FD:42F,460:460,462:462,464:464,466:466,468:468,46A:46A,46C:46C,46E:46E,470:470,472:472,474:474,476:476,478:478,47A:47A,47C:47C,47E:47E,480:480,48A:48A,48C:48C,48E:48E,490:490,492:492,494:494,496:496,498:498,49A:49A,49C:49C,49E:49E,4A0:4A0,4A2:4A2,4A4:4A4,4A6:4A6,4A8:4A8,4AA:4AA,4AC:4AC,4AE:4AE,4B0:4B0,4B2:4B2,4B4:4B4,4B6:4B6,4B8:4B8,4BA:4BA,4BC:4BC,4BE:4BE,4C0:4C1,4C3:4C3,4C5:4C5,4C7:4C7,4C9:4C9,4CB:4CB,4CD:4CD,4D0:4D0,4D2:4D2,4D4:4D4,4D6:4D6,4D8:4D8,4DA:4DA,4DC:4DC,4DE:4DE,4E0:4E0,4E2:4E2,4E4:4E4,4E6:4E6,4E8:4E8,4EA:4EA,4EC:4EC,4EE:4EE,4F0:4F0,4F2:4F2,4F4:4F4,4F6:4F6,4F8:4F8,4FA:4FA,4FC:4FC,4FE:4FE,500:500,502:502,504:504,506:506,508:508,50A:50A,50C:50C,50E:50E,510:510,512:512,514:514,516:516,518:518,51A:51A,51C:51C,51E:51E,520:520,522:522,524:524,526:526,528:528,52A:52A,52C:52C,52E:52E,531:556,10A0:10C5,10C7:10C7,10CD:10CD,13A0:13F5,1C89:1C89,1C90:1CBA,1CBD:1CBF,1E00:1E00,1E02:1E02,1E04:1E04,1E06:1E06,1E08:1E08,1E0A:1E0A,1E0C:1E0C,1E0E:1E0E,1E10:1E10,1E12:1E12,1E14:1E14,1E16:1E16,1E18:1E18,1E1A:1E1A,1E1C:1E1C,1E1E:1E1E,1E20:1E20,1E22:1E22,1E24:1E24,1E26:1E26,1E28:1E28,1E2A:1E2A,1E2C:1E2C,1E2E:1E2E,1E30:1E30,1E32:1E32,1E34:1E34,1E36:1E36,1E38:1E38,1E3A:1E3A,1E3C:1E3C,1E3E:1E3E,1E40:1E40,1E42:1E42,1E44:1E44,1E46:1E46,1E48:1E48,1E4A:1E4A,1E4C:1E4C,1E4E:1E4E,1E50:1E50,1E52:1E52,1E54:1E54,1E56:1E56,1E58:1E58,1E5A:1E5A,1E5C:1E5C,1E5E:1E5E,1E60:1E60,1E62:1E62,1E64:1E64,1E66:1E66,1E68:1E68,1E6A:1E6A,1E6C:1E6C,1E6E:1E6E,1E70:1E70,1E72:1E72,1E74:1E74,1E76:1E76,1E78:1E78,1E7A:1E7A,1E7C:1E7C,1E7E:1E7E,1E80:1E80,1E82:1E82,1E84:1E84,1E86:1E86,1E88:1E88,1E8A:1E8A,1E8C:1E8C,1E8E:1E8E,1E90:1E90,1E92:1E92,1E94:1E94,1E9E:1E9E,1EA0:1EA0,1EA2:1EA2,1EA4:1EA4,1EA6:1EA6,1EA8:1EA8,1EAA:1EAA,1EAC:1EAC,1EAE:1EAE,1EB0:1EB0,1EB2:1EB2,1EB4:1EB4,1EB6:1EB6,1EB8:1EB8,1EBA:1EBA,1EBC:1EBC,1EBE:1EBE,1EC0:1EC0,1EC2:1EC2,1EC4:1EC4,1EC6:1EC6,1EC8:1EC8,1ECA:1ECA,1ECC:1ECC,1ECE:1ECE,1ED0:1ED0,1ED2:1ED2,1ED4:1ED4,1ED6:1ED6,1ED8:1ED8,1EDA:1EDA,1EDC:1EDC,1EDE:1EDE,1EE0:1EE0,1EE2:1EE2,1EE4:1EE4,1EE6:1EE6,1EE8:1EE8,1EEA:1EEA,1EEC:1EEC,1EEE:1EEE,1EF0:1EF0,1EF2:1EF2,1EF4:1EF4,1EF6:1EF6,1EF8:1EF8,1EFA:1EFA,1EFC:1EFC,1EFE:1EFE,1F08:1F0F,1F18:1F1D,1F28:1F2F,1F38:1F3F,1F48:1F4D,1F59:1F59,1F5B:1F5B,1F5D:1F5D,1F5F:1F5F,1F68:1F6F,1FB8:1FBB,1FC8:1FCB,1FD8:1FDB,1FE8:1FEC,1FF8:1FFB,2102:2102,2107:2107,210B:210D,2110:2112,2115:2115,2119:211D,2124:2124,2126:2126,2128:2128,212A:212D,2130:2133,213E:213F,2145:2145,2160:216F,2183:2183,24B6:24CF,2C00:2C2F,2C60:2C60,2C62:2C64,2C67:2C67,2C69:2C69,2C6B:2C6B,2C6D:2C70,2C72:2C72,2C75:2C75,2C7E:2C80,2C82:2C82,2C84:2C84,2C86:2C86,2C88:2C88,2C8A:2C8A,2C8C:2C8C,2C8E:2C8E,2C90:2C90,2C92:2C92,2C94:2C94,2C96:2C96,2C98:2C98,2C9A:2C9A,2C9C:2C9C,2C9E:2C9E,2CA0:2CA0,2CA2:2CA2,2CA4:2CA4,2CA6:2CA6,2CA8:2CA8,2CAA:2CAA,2CAC:2CAC,2CAE:2CAE,2CB0:2CB0,2CB2:2CB2,2CB4:2CB4,2CB6:2CB6,2CB8:2CB8,2CBA:2CBA,2CBC:2CBC,2CBE:2CBE,2CC0:2CC0,2CC2:2CC2,2CC4:2CC4,2CC6:2CC6,2CC8:2CC8,2CCA:2CCA,2CCC:2CCC,2CCE:2CCE,2CD0:2CD0,2CD2:2CD2,2CD4:2CD4,2CD6:2CD6,2CD8:2CD8,2CDA:2CDA,2CDC:2CDC,2CDE:2CDE,2CE0:2CE0,2CE2:2CE2,2CEB:2CEB,2CED:2CED,2CF2:2CF2,A640:A640,A642:A642,A644:A644,A646:A646,A648:A648,A64A:A64A,A64C:A64C,A64E:A64E,A650:A650,A652:A652,A654:A654,A656:A656,A658:A658,A65A:A65A,A65C:A65C,A65E:A65E,A660:A660,A662:A662,A664:A664,A666:A666,A668:A668,A66A:A66A,A66C:A66C,A680:A680,A682:A682,A684:A684,A686:A686,A688:A688,A68A:A68A,A68C:A68C,A68E:A68E,A690:A690,A692:A692,A694:A694,A696:A696,A698:A698,A69A:A69A,A722:A722,A724:A724,A726:A726,A728:A728,A72A:A72A,A72C:A72C,A72E:A72E,A732:A732,A734:A734,A736:A736,A738:A738,A73A:A73A,A73C:A73C,A73E:A73E,A740:A740,A742:A742,A744:A744,A746:A746,A748:A748,A74A:A74A,A74C:A74C,A74E:A74E,A750:A750,A752:A752,A754:A754,A756:A756,A758:A758,A75A:A75A,A75C:A75C,A75E:A75E,A760:A760,A762:A762,A764:A764,A766:A766,A768:A768,A76A:A76A,A76C:A76C,A76E:A76E,A779:A779,A77B:A77B,A77D:A77E,A780:A780,A782:A782,A784:A784,A786:A786,A78B:A78B,A78D:A78D,A790:A790,A792:A792,A796:A796,A798:A798,A79A:A79A,A79C:A79C,A79E:A79E,A7A0:A7A0,A7A2:A7A2,A7A4:A7A4,A7A6:A7A6,A7A8:A7A8,A7AA:A7AE,A7B0:A7B4,A7B6:A7B6,A7B8:A7B8,A7BA:A7BA,A7BC:A7BC,A7BE:A7BE,A7C0:A7C0,A7C2:A7C2,A7C4:A7C7,A7C9:A7C9,A7CB:A7CC,A7D0:A7D0,A7D6:A7D6,A7D8:A7D8,A7DA:A7DA,A7DC:A7DC,A7F5:A7F5,FF21:FF3A,10400:10427,104B0:104D3,10570:1057A,1057C:1058A,1058C:10592,10594:10595,10C80:10CB2,10D50:10D65,118A0:118BF,16E40:16E5F,1D400:1D419,1D434:1D44D,1D468:1D481,1D49C:1D49C,1D49E:1D49F,1D4A2:1D4A2,1D4A5:1D4A6,1D4A9:1D4AC,1D4AE:1D4B5,1D4D0:1D4E9,1D504:1D505,1D507:1D50A,1D50D:1D514,1D516:1D51C,1D538:1D539,1D53B:1D53E,1D540:1D544,1D546:1D546,1D54A:1D550,1D56C:1D585,1D5A0:1D5B9,1D5D4:1D5ED,1D608:1D621,1D63C:1D655,1D670:1D689,1D6A8:1D6C0,1D6E2:1D6FA,1D71C:1D734,1D756:1D76E,1D790:1D7A8,1D7CA:1D7CA,1E900:1E921,1F130:1F149,1F150:1F169,1F170:1F189 +is_titlecase 1C5:1C5,1C8:1C8,1CB:1CB,1F2:1F2,1F88:1F8F,1F98:1F9F,1FA8:1FAF,1FBC:1FBC,1FCC:1FCC,1FFC:1FFC +is_cased 41:5A,61:7A,B5:B5,C0:D6,D8:F6,F8:137,139:18C,18E:1A9,1AC:1B9,1BC:1BD,1BF:1BF,1C4:220,222:233,23A:254,256:257,259:259,25B:25C,260:261,263:266,268:26C,26F:26F,271:272,275:275,27D:27D,280:280,282:283,287:28C,292:292,29D:29E,345:345,370:373,376:377,37B:37D,37F:37F,386:386,388:38A,38C:38C,38E:3A1,3A3:3D1,3D5:3F5,3F7:3FB,3FD:481,48A:52F,531:556,561:587,10A0:10C5,10C7:10C7,10CD:10CD,10D0:10FA,10FD:10FF,13A0:13F5,13F8:13FD,1C80:1C8A,1C90:1CBA,1CBD:1CBF,1D79:1D79,1D7D:1D7D,1D8E:1D8E,1E00:1E9B,1E9E:1E9E,1EA0:1F15,1F18:1F1D,1F20:1F45,1F48:1F4D,1F50:1F57,1F59:1F59,1F5B:1F5B,1F5D:1F5D,1F5F:1F7D,1F80:1FB4,1FB6:1FBC,1FBE:1FBE,1FC2:1FC4,1FC6:1FCC,1FD0:1FD3,1FD6:1FDB,1FE0:1FEC,1FF2:1FF4,1FF6:1FFC,2126:2126,212A:212B,2132:2132,214E:214E,2160:217F,2183:2184,24B6:24E9,2C00:2C70,2C72:2C73,2C75:2C76,2C7E:2CE3,2CEB:2CEE,2CF2:2CF3,2D00:2D25,2D27:2D27,2D2D:2D2D,A640:A66D,A680:A69B,A722:A72F,A732:A76F,A779:A787,A78B:A78D,A790:A794,A796:A7AE,A7B0:A7CD,A7D0:A7D1,A7D6:A7DC,A7F5:A7F6,AB53:AB53,AB70:ABBF,FB00:FB06,FB13:FB17,FF21:FF3A,FF41:FF5A,10400:1044F,104B0:104D3,104D8:104FB,10570:1057A,1057C:1058A,1058C:10592,10594:10595,10597:105A1,105A3:105B1,105B3:105B9,105BB:105BC,10C80:10CB2,10CC0:10CF2,10D50:10D65,10D70:10D85,118A0:118DF,16E40:16E7F,1E900:1E943 diff --git a/crates/unicode/tests/data/version_skew_cpython3.14.txt b/crates/unicode/tests/data/version_skew_cpython3.14.txt index acdb26089ba..e135c422fb4 100644 --- a/crates/unicode/tests/data/version_skew_cpython3.14.txt +++ b/crates/unicode/tests/data/version_skew_cpython3.14.txt @@ -2,6 +2,9 @@ # and the Rust std / icu4x build used here (a later Unicode release assigns them). # Regenerate with RUSTPYTHON_UNICODE_REGEN_SKEW=1. # Format: `predicate start:end,...` with inclusive hex ranges. +is_cased AA:AA,BA:BA,138:138,18D:18D,1AA:1AB,1BA:1BA,1BE:1BE,221:221,234:239,255:255,258:258,25A:25A,25D:25F,262:262,267:267,26D:26E,270:270,273:274,276:27C,27E:27F,281:281,284:286,28D:291,293:293,296:29C,29F:2B8,2C0:2C1,2E0:2E4,37A:37A,3D2:3D4,3FC:3FC,560:560,588:588,10FC:10FC,1D00:1D78,1D7A:1D7C,1D7E:1D8D,1D8F:1DBF,1E9C:1E9D,1E9F:1E9F,2071:2071,207F:207F,2090:209C,2102:2102,2107:2107,210A:2113,2115:2115,2119:211D,2124:2124,2128:2128,212C:212D,212F:2131,2133:2134,2139:2139,213C:213F,2145:2149,2C71:2C71,2C74:2C74,2C77:2C7D,2CE4:2CE4,A69C:A69D,A730:A731,A770:A778,A78E:A78E,A795:A795,A7AF:A7AF,A7CE:A7CF,A7D2:A7D5,A7F1:A7F4,A7F8:A7FA,AB30:AB52,AB54:AB5A,AB5C:AB69,10780:10780,10783:10785,10787:107B0,107B2:107BA,16EA0:16EB8,16EBB:16ED3,1D400:1D454,1D456:1D49C,1D49E:1D49F,1D4A2:1D4A2,1D4A5:1D4A6,1D4A9:1D4AC,1D4AE:1D4B9,1D4BB:1D4BB,1D4BD:1D4C3,1D4C5:1D505,1D507:1D50A,1D50D:1D514,1D516:1D51C,1D51E:1D539,1D53B:1D53E,1D540:1D544,1D546:1D546,1D54A:1D550,1D552:1D6A5,1D6A8:1D6C0,1D6C2:1D6DA,1D6DC:1D6FA,1D6FC:1D714,1D716:1D734,1D736:1D74E,1D750:1D76E,1D770:1D788,1D78A:1D7A8,1D7AA:1D7C2,1D7C4:1D7CB,1DF00:1DF09,1DF0B:1DF1E,1DF25:1DF2A,1E030:1E06D,1F130:1F149,1F150:1F169,1F170:1F189 +is_lowercase 295:295,A7CF:A7CF,A7F1:A7F1,16EBB:16ED3 +is_uppercase A7CE:A7CE,A7D2:A7D2,A7D4:A7D4,16EA0:16EB8 isalnum 88F:88F,C5C:C5C,CDC:CDC,A7CE:A7CF,A7D2:A7D2,A7D4:A7D4,A7F1:A7F1,10940:10959,10EC5:10EC7,11DB0:11DDB,11DE0:11DE9,16EA0:16EB8,16EBB:16ED3,16FF2:16FF6,187F8:187FF,18D09:18D1E,18D80:18DF2,1E6C0:1E6DE,1E6E0:1E6E2,1E6E4:1E6E5,1E6E7:1E6ED,1E6F0:1E6F4,1E6FE:1E6FF,2B73A:2B73F,2CEA2:2CEAD,323B0:33479 isalpha 88F:88F,C5C:C5C,CDC:CDC,A7CE:A7CF,A7D2:A7D2,A7D4:A7D4,A7F1:A7F1,10940:10959,10EC5:10EC7,11DB0:11DDB,16EA0:16EB8,16EBB:16ED3,16FF2:16FF3,187F8:187FF,18D09:18D1E,18D80:18DF2,1E6C0:1E6DE,1E6E0:1E6E2,1E6E4:1E6E5,1E6E7:1E6ED,1E6F0:1E6F4,1E6FE:1E6FF,2B73A:2B73F,2CEA2:2CEAD,323B0:33479 isdecimal 11DE0:11DE9 diff --git a/crates/unicode/tests/data/version_skew_mappings_cpython3.14.txt b/crates/unicode/tests/data/version_skew_mappings_cpython3.14.txt new file mode 100644 index 00000000000..8992b18989c --- /dev/null +++ b/crates/unicode/tests/data/version_skew_mappings_cpython3.14.txt @@ -0,0 +1,5 @@ +# Code points whose simple case mapping differs between CPython 3.14 +# (Unicode 16.0.0) and the icu4x build used here (a later Unicode release +# assigns the pair). Regenerate with RUSTPYTHON_UNICODE_REGEN_SKEW=1. +# Format: `mapping start:end,...` with inclusive hex ranges. +tolower A7CE:A7CE,A7D2:A7D2,A7D4:A7D4,16EA0:16EB8 diff --git a/crates/unicode/tests/differential.rs b/crates/unicode/tests/differential.rs index fef3451fdb9..1bed13e7faf 100644 --- a/crates/unicode/tests/differential.rs +++ b/crates/unicode/tests/differential.rs @@ -11,17 +11,21 @@ //! Both data files use the same run-length format: one `predicate` line per //! str method, followed by comma-separated hex `start:end` inclusive ranges. +// spell-checker:ignore recategorized recategorizations + #[cfg(test)] mod tests { extern crate alloc; use alloc::collections::{BTreeMap, BTreeSet}; - use rustpython_unicode::classify; + use rustpython_unicode::{case, classify}; const MAX: u32 = 0x110000; const REFERENCE: &str = include_str!("data/cpython3.14_predicates.txt"); const VERSION_SKEW: &str = include_str!("data/version_skew_cpython3.14.txt"); + const MAPPINGS: &str = include_str!("data/cpython3.14_mappings.txt"); + const MAPPING_SKEW: &str = include_str!("data/version_skew_mappings_cpython3.14.txt"); fn crate_predicate(name: &str, cp: u32) -> bool { let Some(c) = char::from_u32(cp) else { @@ -41,10 +45,25 @@ mod tests { // is "may start an identifier". classify_is_identifier_char(c) } + "is_lowercase" => case::is_lowercase(c), + "is_uppercase" => case::is_uppercase(c), + "is_titlecase" => case::is_titlecase(c), + "is_cased" => case::is_cased(c), other => panic!("unknown predicate {other}"), } } + /// The crate's simple case mapping for `name` at `cp`, as a code point. + fn crate_mapping(name: &str, cp: u32) -> u32 { + let Some(c) = char::from_u32(cp) else { + return cp; + }; + match name { + "tolower" => case::simple_lowercase(c) as u32, + other => panic!("unknown mapping {other}"), + } + } + fn classify_is_identifier_char(c: char) -> bool { rustpython_unicode::identifier::is_start(c) } @@ -80,6 +99,36 @@ mod tests { map } + /// Parse a `name -> {code point -> mapped code point}` table. + /// + /// Each non-comment line is `name cp:mapped,cp:mapped,...` listing only the + /// code points whose mapping differs from identity. + fn parse_mappings(text: &str) -> BTreeMap> { + let mut map = BTreeMap::new(); + for line in text.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let (name, packed) = match line.split_once(' ') { + Some((name, packed)) => (name, packed.trim()), + None => (line, ""), + }; + let mut table = BTreeMap::new(); + if !packed.is_empty() { + for pair in packed.split(',') { + let (cp, mapped) = pair.split_once(':').expect("pair is cp:mapped"); + table.insert( + u32::from_str_radix(cp, 16).unwrap(), + u32::from_str_radix(mapped, 16).unwrap(), + ); + } + } + map.insert(name.to_string(), table); + } + map + } + /// Collapse a sorted code-point set into inclusive `start:end` runs. fn encode_ranges(set: &BTreeSet) -> String { let mut runs = Vec::new(); @@ -119,14 +168,23 @@ mod tests { out } + /// Documented `cpython=true/crate=false` divergences: a code point that had a + /// property in Unicode 16.0.0 but lost it in the release icu4x ships, because + /// the code point was recategorized (not a regression in this crate). + /// + /// * U+0295 LATIN LETTER PHARYNGEAL VOICED FRICATIVE was general category `Ll` + /// in Unicode 16.0.0 and `Lo` from 17.0.0, so it is no longer `Lowercase`. + const KNOWN_RECATEGORIZATIONS: &[(&str, u32)] = &[("is_lowercase", 0x0295)]; + /// Regenerate `data/version_skew_cpython3.14.txt` from the current toolchain. /// /// Run with `RUSTPYTHON_UNICODE_REGEN_SKEW=1 cargo test -p rustpython-unicode - /// --test differential` after bumping the Rust/icu toolchain. All divergences - /// must be one-directional (crate=true, cpython=false) — newly-assigned code + /// --test differential` after bumping the Rust/icu toolchain. Divergences are + /// normally one-directional (crate=true, cpython=false) — newly-assigned code /// points from a later Unicode release. A `cpython=true, crate=false` entry - /// means a code point lost a property, which is a real regression, so this - /// refuses to record it. + /// means a code point lost a property; that is a real regression unless it is + /// an explicit entry in `KNOWN_RECATEGORIZATIONS`, so this refuses to record + /// any other reverse-direction divergence. #[test] fn regen_version_skew() { if std::env::var_os("RUSTPYTHON_UNICODE_REGEN_SKEW").is_none() { @@ -137,7 +195,9 @@ mod tests { let regressions: Vec<_> = divergences .iter() - .filter(|(_, _, expected)| *expected) + .filter(|(name, cp, expected)| { + *expected && !KNOWN_RECATEGORIZATIONS.contains(&(name.as_str(), *cp)) + }) .collect(); assert!( regressions.is_empty(), @@ -232,4 +292,133 @@ mod tests { panic!("{msg}"); } } + + /// All `(mapping, code point)` where the crate and CPython map differently. + fn all_mapping_divergences( + reference: &BTreeMap>, + ) -> Vec<(String, u32)> { + let mut out = Vec::new(); + for (name, table) in reference { + for cp in 0..MAX { + let expected = table.get(&cp).copied().unwrap_or(cp); + if crate_mapping(name, cp) != expected { + out.push((name.clone(), cp)); + } + } + } + out + } + + /// Regenerate `data/version_skew_mappings_cpython3.14.txt` from the current + /// toolchain (`RUSTPYTHON_UNICODE_REGEN_SKEW=1`). + /// + /// Divergences are normally the crate gaining a mapping a later Unicode + /// release assigns. A code point that CPython maps but the crate leaves + /// unmapped is a regression, not version skew, so this refuses to record it. + #[test] + fn regen_mapping_version_skew() { + if std::env::var_os("RUSTPYTHON_UNICODE_REGEN_SKEW").is_none() { + return; + } + let reference = parse_mappings(MAPPINGS); + let divergences = all_mapping_divergences(&reference); + + let regressions: Vec<_> = divergences + .iter() + .filter(|(name, cp)| { + let expected = reference.get(name).and_then(|t| t.get(cp)).copied(); + expected.is_some_and(|e| e != *cp) && crate_mapping(name, *cp) == *cp + }) + .collect(); + assert!( + regressions.is_empty(), + "refusing to record {} cpython-maps/crate-unmapped divergence(s) — these are \ + regressions, not version skew: {:?}", + regressions.len(), + ®ressions[..regressions.len().min(20)] + ); + + let mut by_mapping: BTreeMap> = BTreeMap::new(); + for (name, cp) in &divergences { + by_mapping.entry(name.clone()).or_default().insert(*cp); + } + + let mut body = String::from( + "# Code points whose simple case mapping differs between CPython 3.14\n\ + # (Unicode 16.0.0) and the icu4x build used here (a later Unicode release\n\ + # assigns the pair). Regenerate with RUSTPYTHON_UNICODE_REGEN_SKEW=1.\n\ + # Format: `mapping start:end,...` with inclusive hex ranges.\n", + ); + for (name, set) in &by_mapping { + body.push_str(&format!("{name} {}\n", encode_ranges(set))); + } + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/data/version_skew_mappings_cpython3.14.txt" + ); + std::fs::write(path, body).unwrap(); + eprintln!( + "wrote {} skew code points across {} mappings to {path}", + divergences.len(), + by_mapping.len() + ); + } + + #[test] + fn simple_mappings_match_cpython_except_documented_version_skew() { + let reference = parse_mappings(MAPPINGS); + let skew = parse_ranges(MAPPING_SKEW); + + let allowed = |name: &str, cp: u32| skew.get(name).is_some_and(|set| set.contains(&cp)); + + let mut unexpected: Vec<(String, u32, u32, u32)> = Vec::new(); + for (name, table) in &reference { + for cp in 0..MAX { + let expected = table.get(&cp).copied().unwrap_or(cp); + let actual = crate_mapping(name, cp); + if expected != actual && !allowed(name, cp) { + unexpected.push((name.clone(), cp, expected, actual)); + } + } + } + + let mut stale: Vec<(String, u32)> = Vec::new(); + for (name, set) in &skew { + for &cp in set { + let expected = reference + .get(name) + .and_then(|t| t.get(&cp)) + .copied() + .unwrap_or(cp); + if crate_mapping(name, cp) == expected { + stale.push((name.clone(), cp)); + } + } + } + + if !unexpected.is_empty() || !stale.is_empty() { + let mut msg = String::new(); + if !unexpected.is_empty() { + msg.push_str(&format!( + "{} undocumented mapping divergence(s) from CPython:\n", + unexpected.len() + )); + for (name, cp, expected, actual) in unexpected.iter().take(50) { + msg.push_str(&format!( + " {name} U+{cp:04X}: cpython=U+{expected:04X} crate=U+{actual:04X}\n" + )); + } + } + if !stale.is_empty() { + msg.push_str(&format!( + "{} stale version_skew_mappings_cpython3.14.txt entries that now agree:\n", + stale.len() + )); + for (name, cp) in stale.iter().take(50) { + msg.push_str(&format!(" {name} U+{cp:04X}\n")); + } + } + panic!("{msg}"); + } + } } diff --git a/crates/unicode/tests/generate_reference.py b/crates/unicode/tests/generate_reference.py index c7b21e66fc4..6a78230de75 100644 --- a/crates/unicode/tests/generate_reference.py +++ b/crates/unicode/tests/generate_reference.py @@ -14,6 +14,7 @@ from __future__ import annotations +import _sre import pathlib import sys import unicodedata @@ -32,6 +33,29 @@ "isidentifier": str.isidentifier, } +# Casing predicates, keyed by the crate function each one exercises. Each is +# sourced from the exact property that function computes: +# * is_lowercase / is_uppercase mirror Py_UNICODE_ISLOWER / ISUPPER, which for a +# single character are str.islower() / str.isupper(). +# * is_titlecase is the Lt general category (NOT str.istitle(), which also +# reports plain uppercase letters as titlecased). +# * is_cased is the Cased property (Py_UNICODE_ISCASED), via _sre. +CASE_PREDICATES = { + "is_lowercase": lambda c: c.islower(), + "is_uppercase": lambda c: c.isupper(), + "is_titlecase": lambda c: unicodedata.category(c) == "Lt", + "is_cased": lambda c: _sre.unicode_iscased(ord(c)), +} + +# Simple one-to-one lowercase mapping (Py_UNICODE_TOLOWER via _sre). This is the +# mapping the regex IGNORECASE path depends on. Emitted as `cp:mapping,...` for +# code points that map to something other than themselves. CPython exposes no +# Python-level simple-uppercase oracle (_sre has unicode_tolower only), so +# toupper is left to the SRE unit tests. +CASE_MAPPINGS = { + "tolower": _sre.unicode_tolower, +} + def encode_ranges(is_true) -> list[tuple[int, int]]: """Collapse the true-set of ``is_true`` into inclusive ``[start, end]`` runs.""" @@ -56,17 +80,33 @@ def main() -> int: "expected 16.0.0 (CPython 3.14); regenerating anyway\n" ) - out = pathlib.Path(__file__).parent / "data" / "cpython3.14_predicates.txt" - out.parent.mkdir(parents=True, exist_ok=True) + data = pathlib.Path(__file__).parent / "data" + data.mkdir(parents=True, exist_ok=True) lines = [f"# unidata_version {unicodedata.unidata_version}"] for name, method in STR_PREDICATES.items(): ranges = encode_ranges(lambda cp, m=method: m(chr(cp))) packed = ",".join(f"{s:X}:{e:X}" for s, e in ranges) lines.append(f"{name} {packed}") + for name, method in CASE_PREDICATES.items(): + ranges = encode_ranges(lambda cp, m=method: m(chr(cp))) + packed = ",".join(f"{s:X}:{e:X}" for s, e in ranges) + lines.append(f"{name} {packed}") + + predicates = data / "cpython3.14_predicates.txt" + predicates.write_text("\n".join(lines) + "\n") + print(f"wrote {predicates} ({predicates.stat().st_size} bytes)") + + mapping_lines = [f"# unidata_version {unicodedata.unidata_version}"] + for name, method in CASE_MAPPINGS.items(): + pairs = [ + f"{cp:X}:{mapped:X}" for cp in range(MAX) if (mapped := method(cp)) != cp + ] + mapping_lines.append(f"{name} {','.join(pairs)}") - out.write_text("\n".join(lines) + "\n") - print(f"wrote {out} ({out.stat().st_size} bytes)") + mappings = data / "cpython3.14_mappings.txt" + mappings.write_text("\n".join(mapping_lines) + "\n") + print(f"wrote {mappings} ({mappings.stat().st_size} bytes)") return 0 diff --git a/crates/vm/Cargo.toml b/crates/vm/Cargo.toml index 24006c8b3b9..ca92b9dfd76 100644 --- a/crates/vm/Cargo.toml +++ b/crates/vm/Cargo.toml @@ -82,12 +82,6 @@ optional = { workspace = true } result-like = { workspace = true } timsort = { workspace = true } -## unicode stuff -icu_casemap = { workspace = true } -icu_locale = { workspace = true } -icu_properties = { workspace = true } -writeable = { workspace = true } - [target.'cfg(unix)'.dependencies] exitcode = { workspace = true } diff --git a/crates/vm/src/anystr.rs b/crates/vm/src/anystr.rs index 3aa40eaf3a2..69ba525267a 100644 --- a/crates/vm/src/anystr.rs +++ b/crates/vm/src/anystr.rs @@ -1,9 +1,7 @@ use core::ops::Range; -use icu_properties::props::{ - BinaryProperty, EnumeratedProperty, GeneralCategory, GeneralCategoryGroup, -}; use num_traits::{cast::ToPrimitive, sign::Signed}; +use rustpython_unicode::case; use crate::{ Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, @@ -448,24 +446,18 @@ pub(crate) trait AnyStr { // Unified form of CPython functions: // unicode_isupper_impl // unicode_islower_impl - fn is_cased(&self) -> bool - where - VALID: BinaryProperty, - INVALID: BinaryProperty, - { + fn is_cased(&self, valid: fn(char) -> bool, invalid: fn(char) -> bool) -> bool { let mut all_cased = false; for c in self .as_bytes() .utf8_chunks() .flat_map(|c| c.valid().chars()) { - if INVALID::for_char(c) - || GeneralCategoryGroup::TitlecaseLetter.contains(GeneralCategory::for_char(c)) - { + if invalid(c) || case::is_titlecase(c) { return false; } - if !all_cased && VALID::for_char(c) { + if !all_cased && valid(c) { all_cased = true; } } diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index a72a272679b..b1a119f5ef2 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -30,7 +30,6 @@ use crate::{ AsMapping, AsNumber, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, }, - utils::VecFmtWriter, }; use alloc::{borrow::Cow, fmt}; use ascii::{AsciiChar, AsciiStr, AsciiString}; @@ -48,14 +47,7 @@ use rustpython_common::{ wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Concat}, }; -use icu_casemap::TitlecaseMapper; -use icu_locale::LanguageIdentifier; -use icu_properties::props::{ - BinaryProperty, CaseIgnorable, Cased, EnumeratedProperty, GeneralCategory, - GeneralCategoryGroup, Lowercase, Uppercase, -}; -use rustpython_unicode as unicode; -use writeable::Writeable; +use rustpython_unicode::{self as unicode, case}; impl<'a> TryFromBorrowedObject<'a> for String { fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { @@ -778,40 +770,8 @@ impl PyStr { } s.into() } - PyKindStr::Utf8(s) => { - let mut chars = s.char_indices(); - let mut out = VecFmtWriter(Vec::with_capacity(s.len())); - titlecase_first(s, &mut chars, &mut out); - for (i, ch) in chars { - lowercase_or_sigma(ch, s, i, &mut out); - } - unsafe { Wtf8Buf::from_bytes_unchecked(out.0) } - } - PyKindStr::Wtf8(s) => { - let mut out = VecFmtWriter(Vec::with_capacity(s.len())); - let mut chunks = s.as_bytes().utf8_chunks(); - - if let Some(first) = chunks.next() { - let s = first.valid(); - let mut chars = s.char_indices(); - titlecase_first(s, &mut chars, &mut out); - for (i, ch) in chars { - lowercase_or_sigma(ch, s, i, &mut out); - } - out.0.extend(first.invalid()); - } - // This loop is only hit if the WTF-8 buffer contains invalid Unicode. Otherwise, - // everything is handled above without chunking. - for chunk in chunks { - let s = chunk.valid(); - for (i, ch) in s.char_indices() { - lowercase_or_sigma(ch, s, i, &mut out); - } - out.0.extend(chunk.invalid()); - } - - unsafe { Wtf8Buf::from_bytes_unchecked(out.0) } - } + PyKindStr::Utf8(s) => case::capitalize_str(s).into(), + PyKindStr::Wtf8(s) => case::capitalize_wtf8(s), } } @@ -1062,23 +1022,8 @@ impl PyStr { PyKindStr::Ascii(_) => unsafe { Wtf8Buf::from_bytes_unchecked(title_ascii(self.as_bytes())) }, - PyKindStr::Utf8(s) => { - let mut out = VecFmtWriter(Vec::with_capacity(s.len())); - titlecase_string(s, &mut out); - // SAFETY: `s` is valid UTF-8 and titlecase_string only works on Unicode. - unsafe { Wtf8Buf::from_bytes_unchecked(out.0) } - } - PyKindStr::Wtf8(s) => { - let mut out = VecFmtWriter(Vec::with_capacity(s.len())); - for chunk in s.as_bytes().utf8_chunks() { - titlecase_string(chunk.valid(), &mut out); - out.0.extend(chunk.invalid()); - } - // SAFETY: - // * `s` is valid WTF-8; surrogate bytes were appended without processing. - // * TitlecaseMapper produces valid UTF-8. - unsafe { Wtf8Buf::from_bytes_unchecked(out.0) } - } + PyKindStr::Utf8(s) => case::title_str(s).into(), + PyKindStr::Wtf8(s) => case::title_wtf8(s), } } @@ -1089,23 +1034,8 @@ impl PyStr { // SAFETY: ASCII is valid Unicode and swapcase_ascii does not produce non-ASCII. Wtf8Buf::from_bytes_unchecked(swapcase_ascii(s.as_bytes())) }, - PyKindStr::Utf8(s) => { - let mut out = VecFmtWriter(Vec::with_capacity(s.len())); - swapcase_utf8(s, &mut out); - // SAFETY: `s` is valid UTF-8 and swapcase_utf8 only works on Unicode. - unsafe { Wtf8Buf::from_bytes_unchecked(out.0) } - } - PyKindStr::Wtf8(s) => { - let mut out = VecFmtWriter(Vec::with_capacity(s.len())); - for chunk in s.as_bytes().utf8_chunks() { - swapcase_utf8(chunk.valid(), &mut out); - out.0.extend(chunk.invalid()); - } - // SAFETY: - // * `s` is valid WTF-8; surrogate bytes were appended without processing. - // * swapcase_utf8 produces valid UTF-8. - unsafe { Wtf8Buf::from_bytes_unchecked(out.0) } - } + PyKindStr::Utf8(s) => case::swapcase_str(s).into(), + PyKindStr::Wtf8(s) => case::swapcase_wtf8(s), } } @@ -1307,9 +1237,7 @@ impl PyStr { let mut cased = false; let mut previous_is_cased = false; for c in self.as_wtf8().code_points().map(CodePoint::to_char_lossy) { - if c.is_uppercase() - || GeneralCategoryGroup::TitlecaseLetter.contains(GeneralCategory::for_char(c)) - { + if c.is_uppercase() || case::is_titlecase(c) { if previous_is_cased { return false; } @@ -1546,102 +1474,6 @@ impl PyStr { } } -/// Title case first char if it is cased or write as is. -/// -/// This matches CPython's behavior: -/// "123abc" -> "123abc" -/// "abc" -> "Abc" -fn titlecase_first(s: &str, chars: &mut core::str::CharIndices<'_>, out: &mut VecFmtWriter) { - if let Some((first_pos, first_ch)) = chars.next() { - let first = &s[..first_pos + first_ch.len_utf8()]; - let tm = TitlecaseMapper::new(); - tm.titlecase_segment(first, &LanguageIdentifier::UNKNOWN, Default::default()) - .write_to(out) - .expect("Writing to an in-memory buffer cannot fail."); - } -} - -/// Title case a string following CPython conventions. -/// -/// CPython title cases each char in a segment. A "segment" is split by case ignorable characters -/// rather than whitespace. -/// "123abc" -> "123Abc" -/// "123abc456def" -> "123Abc456Def" -/// "123 abc" -> "123 Abc" -fn titlecase_string(s: &str, out: &mut VecFmtWriter) { - let mut previous_is_cased = false; - let mapper = TitlecaseMapper::new(); - for (i, ch) in s.char_indices() { - if previous_is_cased { - lowercase_or_sigma(ch, s, i, out); - } else { - let s = &s[i..i + ch.len_utf8()]; - mapper - .titlecase_segment(s, &LanguageIdentifier::UNKNOWN, Default::default()) - .write_to(out) - .expect("Writing to an in-memory buffer cannot fail."); - } - - previous_is_cased = Cased::for_char(ch); - } -} - -fn lowercase_or_sigma(ch: char, s: &str, i: usize, out: &mut VecFmtWriter) { - let sigma = 'Σ'; - if ch == sigma { - let sigma_cased = handle_capital_sigma(s, i); - let mut buf = [0u8; 4]; - let s = sigma_cased.encode_utf8(&mut buf); - out.0.extend(s.as_bytes()); - } else { - for ch in ch.to_lowercase() { - let mut buf = [0u8; 4]; - let s = ch.encode_utf8(&mut buf); - out.0.extend(s.as_bytes()); - } - } -} - -// Handle context-sensitive sigma. -// -// CPython handles sigma as a special case. This is more efficient than using icu4x to scan the -// entire string with CaseMapper because CaseMapper would allocate to produce a new string. The -// icu4x crates are robust but CPython's capitalize() is NOT so we can skip the extra allocs. -fn handle_capital_sigma(s: &str, i: usize) -> char { - let (left, rest) = s.split_at(i); - let right = &rest['Σ'.len_utf8()..]; - - // Check if any chars before or after sigma are cased. - let before = left - .chars() - .rev() - .find(|&ch| !CaseIgnorable::for_char(ch)) - .is_some_and(Cased::for_char); - let after = right - .chars() - .find(|&ch| !CaseIgnorable::for_char(ch)) - .is_some_and(Cased::for_char); - if before && !after { 'ς' } else { 'σ' } -} - -fn swapcase_utf8(s: &str, out: &mut VecFmtWriter) { - for (i, ch) in s.char_indices() { - if ch.is_uppercase() { - lowercase_or_sigma(ch, s, i, out); - } else if ch.is_lowercase() { - for ch in ch.to_uppercase() { - let mut buf = [0u8; 4]; - let s = ch.encode_utf8(&mut buf); - out.0.extend(s.as_bytes()); - } - } else { - let mut buf = [0u8; 4]; - let s = ch.encode_utf8(&mut buf); - out.0.extend(s.as_bytes()); - } - } -} - impl PyRef { #[must_use] pub fn is_empty(&self) -> bool { @@ -2454,11 +2286,11 @@ impl AnyStr for str { } fn py_islower(&self) -> bool { - self.is_cased::() + self.is_cased(case::is_lowercase, case::is_uppercase) } fn py_isupper(&self) -> bool { - self.is_cased::() + self.is_cased(case::is_uppercase, case::is_lowercase) } } @@ -2574,11 +2406,11 @@ impl AnyStr for Wtf8 { } fn py_islower(&self) -> bool { - self.is_cased::() + self.is_cased(case::is_lowercase, case::is_uppercase) } fn py_isupper(&self) -> bool { - self.is_cased::() + self.is_cased(case::is_uppercase, case::is_lowercase) } } diff --git a/crates/vm/src/stdlib/_sre.rs b/crates/vm/src/stdlib/_sre.rs index 1f62b48b137..f7bc10e6640 100644 --- a/crates/vm/src/stdlib/_sre.rs +++ b/crates/vm/src/stdlib/_sre.rs @@ -23,7 +23,7 @@ mod _sre { use num_traits::ToPrimitive; use rustpython_sre_engine::{ Request, SearchIter, SreFlag, State, StrDrive, - string::{lower_ascii, lower_unicode, upper_unicode}, + string::{lower_ascii, lower_unicode}, }; #[pyattr] @@ -41,8 +41,7 @@ mod _sre { #[pyfunction] fn unicode_iscased(ch: i32) -> bool { - let ch = ch as u32; - ch != lower_unicode(ch) || ch != upper_unicode(ch) + char::from_u32(ch as u32).is_some_and(rustpython_unicode::case::is_cased) } #[pyfunction] diff --git a/crates/vm/src/utils.rs b/crates/vm/src/utils.rs index 51e27123fc8..b5117ddd8d1 100644 --- a/crates/vm/src/utils.rs +++ b/crates/vm/src/utils.rs @@ -1,5 +1,3 @@ -use core::fmt; - use rustpython_common::wtf8::{Wtf8, Wtf8Buf}; use crate::{ @@ -74,16 +72,3 @@ where Ok(repr) } - -/// Wrapper around a bytes vector that implements [`fmt::Write`]. -/// -/// # Safety -/// Don't assume the contents of the internal vector are valid UTF-8/WTF-8. -pub(crate) struct VecFmtWriter(pub Vec); - -impl fmt::Write for VecFmtWriter { - fn write_str(&mut self, s: &str) -> fmt::Result { - self.0.extend(s.bytes()); - Ok(()) - } -}