diff --git a/src/rust/src/parser.rs b/src/rust/src/parser.rs index 83c4b98a0..f8d24c5a6 100644 --- a/src/rust/src/parser.rs +++ b/src/rust/src/parser.rs @@ -91,6 +91,16 @@ where } } +/// Sort a word list in the same order as C's `strcasecmp`. The C encoders look words up with +/// `bsearch` (ccx_encoders_helpers.c), which only finds them in a list sorted this way. +fn sort_word_list(list: &mut [String]) { + list.sort_by(|a, b| { + a.bytes() + .map(|c| c.to_ascii_lowercase()) + .cmp(b.bytes().map(|c| c.to_ascii_lowercase())) + }); +} + fn process_word_file(filename: &str, list: &mut Vec) -> Result<(), std::io::Error> { let file = File::open(filename)?; let reader = BufReader::new(file); @@ -1446,6 +1456,8 @@ impl OptionsExt for Options { } } } + sort_word_list(capitalization_list); + sort_word_list(profane); // Init telexcc redundant options tlt_config.dolevdist = self.dolevdist; @@ -2566,6 +2578,54 @@ pub mod tests { assert!(options.enc_cfg.filter_profanity); } + /// Compare two words like C's `strcasecmp`, which the C `bsearch` lookups use. + fn strcasecmp(a: &str, b: &str) -> std::cmp::Ordering { + a.bytes() + .map(|c| c.to_ascii_lowercase()) + .cmp(b.bytes().map(|c| c.to_ascii_lowercase())) + } + + #[test] + fn test_sort_word_list_uses_strcasecmp_order() { + let mut list: Vec = ["june", "Zebra", "_x", "Apple", "banana"] + .iter() + .map(|s| s.to_string()) + .collect(); + sort_word_list(&mut list); + // '_' (0x5f) sorts before the lowercase letters, as with strcasecmp + assert_eq!(list, ["_x", "Apple", "banana", "june", "Zebra"]); + } + + #[test] + fn test_word_lists_are_sorted_for_bsearch() { + let args = Args::try_parse_from(["./ccextractor", "input_file", "--sentencecap", "--kf"]) + .expect("Failed to parse arguments"); + let mut options = Options::default(); + let mut tlt_config = TeletextConfig::default(); + let mut capitalization_list: Vec = vec![]; + let mut profane: Vec = vec![]; + options.parse_parameters( + &args, + &mut tlt_config, + &mut capitalization_list, + &mut profane, + ); + + for list in [&capitalization_list, &profane] { + assert!(!list.is_empty()); + // Every word must be found by a binary search, as C's bsearch does + for word in list.iter() { + assert!( + list.binary_search_by(|w| strcasecmp(w, word)).is_ok(), + "{} not found", + word + ); + } + } + let june = capitalization_list.binary_search_by(|w| strcasecmp(w, "june")); + assert_eq!(june.map(|i| capitalization_list[i].as_str()), Ok("June")); + } + #[test] fn test_profanity_file_sets_filter_file() { let (options, _) = parse_args(&["--profanity-file", "badwords.txt"]);