[FIX] --sentencecap, --capfile, --kf, --profanity-file miss most words (word lists never sorted)

src/rust/src/parser.rs

@@ -91,6 +91,16 @@ where
     }
 }
 
+/// Sort a word list in the same order as C's `strcasecmp`. The C encoders look words up with
+/// `bsearch` (ccx_encoders_helpers.c), which only finds them in a list sorted this way.
+fn sort_word_list(list: &mut [String]) {
+    list.sort_by(|a, b| {
+        a.bytes()
+            .map(|c| c.to_ascii_lowercase())
+            .cmp(b.bytes().map(|c| c.to_ascii_lowercase()))
+    });
+}
+
 fn process_word_file(filename: &str, list: &mut Vec<String>) -> Result<(), std::io::Error> {
     let file = File::open(filename)?;
     let reader = BufReader::new(file);
@@ -1446,6 +1456,8 @@ impl OptionsExt for Options {
                 }
             }
         }
+        sort_word_list(capitalization_list);
+        sort_word_list(profane);
 
         // Init telexcc redundant options
         tlt_config.dolevdist = self.dolevdist;
@@ -2566,6 +2578,54 @@ pub mod tests {
         assert!(options.enc_cfg.filter_profanity);
     }
 
+    /// Compare two words like C's `strcasecmp`, which the C `bsearch` lookups use.
+    fn strcasecmp(a: &str, b: &str) -> std::cmp::Ordering {
+        a.bytes()
+            .map(|c| c.to_ascii_lowercase())
+            .cmp(b.bytes().map(|c| c.to_ascii_lowercase()))
+    }
+
+    #[test]
+    fn test_sort_word_list_uses_strcasecmp_order() {
+        let mut list: Vec<String> = ["june", "Zebra", "_x", "Apple", "banana"]
+            .iter()
+            .map(|s| s.to_string())
+            .collect();
+        sort_word_list(&mut list);
+        // '_' (0x5f) sorts before the lowercase letters, as with strcasecmp
+        assert_eq!(list, ["_x", "Apple", "banana", "june", "Zebra"]);
+    }
+
+    #[test]
+    fn test_word_lists_are_sorted_for_bsearch() {
+        let args = Args::try_parse_from(["./ccextractor", "input_file", "--sentencecap", "--kf"])
+            .expect("Failed to parse arguments");
+        let mut options = Options::default();
+        let mut tlt_config = TeletextConfig::default();
+        let mut capitalization_list: Vec<String> = vec![];
+        let mut profane: Vec<String> = vec![];
+        options.parse_parameters(
+            &args,
+            &mut tlt_config,
+            &mut capitalization_list,
+            &mut profane,
+        );
+
+        for list in [&capitalization_list, &profane] {
+            assert!(!list.is_empty());
+            // Every word must be found by a binary search, as C's bsearch does
+            for word in list.iter() {
+                assert!(
+                    list.binary_search_by(|w| strcasecmp(w, word)).is_ok(),
+                    "{} not found",
+                    word
+                );
+            }
+        }
+        let june = capitalization_list.binary_search_by(|w| strcasecmp(w, "june"));
+        assert_eq!(june.map(|i| capitalization_list[i].as_str()), Ok("June"));
+    }
+
     #[test]
     fn test_profanity_file_sets_filter_file() {
         let (options, _) = parse_args(&["--profanity-file", "badwords.txt"]);