diff --git a/src/uu/wc/Cargo.toml b/src/uu/wc/Cargo.toml index be25c406c5a..241ee11a11c 100644 --- a/src/uu/wc/Cargo.toml +++ b/src/uu/wc/Cargo.toml @@ -24,6 +24,7 @@ thiserror = { workspace = true } unicode-width = { workspace = true } uucore = { workspace = true, features = [ "hardware", + "i18n-charmap", "parser", "pipes", "quoting-style", diff --git a/src/uu/wc/src/count_fast.rs b/src/uu/wc/src/count_fast.rs index 82200fe08d9..0e01eea9620 100644 --- a/src/uu/wc/src/count_fast.rs +++ b/src/uu/wc/src/count_fast.rs @@ -5,6 +5,7 @@ use crate::{wc_simd_allowed, word_count::WordCount}; use uucore::hardware::SimdPolicy; +use uucore::i18n::charmap::{Encoding, locale_encoding}; use super::WordCountable; @@ -196,6 +197,11 @@ pub(crate) fn count_bytes_chars_and_lines_fast< let buf: &mut [u8] = &mut AlignedBuffer::default().data; let policy = SimdPolicy::detect(); let simd_allowed = wc_simd_allowed(policy); + + // In C/POSIX locale, characters are equivalent to bytes (MB_CUR_MAX == 1). + // This follows GNU coreutils behavior. + let chars_are_bytes = locale_encoding() == Encoding::SingleByte; + loop { match handle.read(buf) { Ok(0) => return (total, None), @@ -204,7 +210,9 @@ pub(crate) fn count_bytes_chars_and_lines_fast< total.bytes += n; } if COUNT_CHARS { - total.chars += if simd_allowed { + total.chars += if chars_are_bytes { + n + } else if simd_allowed { bytecount::num_chars(&buf[..n]) } else { bytecount::naive_num_chars(&buf[..n]) diff --git a/src/uu/wc/src/wc.rs b/src/uu/wc/src/wc.rs index 715c22969c7..41b548b5bbb 100644 --- a/src/uu/wc/src/wc.rs +++ b/src/uu/wc/src/wc.rs @@ -25,6 +25,7 @@ use std::{ use clap::{Arg, ArgAction, ArgMatches, Command, builder::ValueParser}; use unicode_width::UnicodeWidthChar; use utf8::{BufReadDecoder, BufReadDecoderError}; +use uucore::i18n::charmap::{Encoding, locale_encoding}; use uucore::{display::Quotable, quoting_style::locale_aware_shell_escape, translate}; use uucore::{ @@ -578,6 +579,7 @@ fn process_chunk< current_len: &mut usize, in_word: &mut bool, is_posixly_correct: bool, + chars_are_bytes: bool, ) { for ch in text.chars() { if SHOW_WORDS { @@ -613,12 +615,17 @@ fn process_chunk< if SHOW_LINES && ch == '\n' { total.lines += 1; } - if SHOW_CHARS { + if SHOW_CHARS && !chars_are_bytes { total.chars += 1; } } total.bytes += text.len(); + // In C/POSIX locale, chars count equals bytes count + if SHOW_CHARS && chars_are_bytes { + total.chars += text.len(); + } + total.max_line_length = max(*current_len, total.max_line_length); } @@ -626,10 +633,17 @@ fn handle_error( error: BufReadDecoderError<'_>, total: &mut WordCount, in_word: &mut bool, + chars_are_bytes: bool, ) -> Option { match error { BufReadDecoderError::InvalidByteSequence(bytes) => { total.bytes += bytes.len(); + // C/POSIX locale: each byte is one char, so invalid bytes count + // (GNU: `printf '\303\251\377' | LC_ALL=C wc -wm` gives `1 3`). + // UTF-8 handling of invalid bytes is unchanged (out of scope). + if chars_are_bytes { + total.chars += bytes.len(); + } if !(*in_word) { *in_word = true; total.words += 1; @@ -654,6 +668,7 @@ fn word_count_from_reader_specialized< let mut in_word = false; let mut current_len = 0; let is_posixly_correct = *IS_POSIXLY_CORRECT; + let chars_are_bytes = SHOW_CHARS && locale_encoding() == Encoding::SingleByte; while let Some(chunk) = reader.next_strict() { match chunk { Ok(text) => { @@ -663,10 +678,11 @@ fn word_count_from_reader_specialized< &mut current_len, &mut in_word, is_posixly_correct, + chars_are_bytes, ); } Err(e) => { - if let Some(e) = handle_error(e, &mut total, &mut in_word) { + if let Some(e) = handle_error(e, &mut total, &mut in_word, chars_are_bytes) { return (total, Some(e)); } } diff --git a/tests/by-util/test_wc.rs b/tests/by-util/test_wc.rs index cdca007b7de..242e55e8ff8 100644 --- a/tests/by-util/test_wc.rs +++ b/tests/by-util/test_wc.rs @@ -3,7 +3,8 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (flags) lwmcL clmwL ; (path) bogusfile emptyfile manyemptylines moby notrailingnewline onelongemptyline onelongword weirdchars ioerrdir +// spell-checker:ignore (flags) lwmcL clmwL ; (path) bogusfile emptyfile manyemptylines moby notrailingnewline onelongemptyline onelongword weirdchars ioerrdir CTYPE +// spell-checker:ignore (Vietnamese) Tiếng Việt chào use rstest::rstest; #[cfg(unix)] @@ -83,6 +84,7 @@ fn test_stdin_explicit() { fn test_wc_fixture(#[case] args: &[&str], #[case] fixture: &str, #[case] expected: &str) { new_ucmd!() .args(args) + .env("LC_ALL", "en_US.UTF-8") .pipe_in_fixture(fixture) .succeeds() .stdout_only(expected); @@ -760,3 +762,126 @@ fn test_posixly_correct_whitespace() { .succeeds() .stdout_is("1\n"); } + +#[test] +fn test_wc_chars_c_locale() { + // In C/POSIX locale each byte is one character, so chars count equals bytes. + // Byte counts: "Tiếng" is 7 (ế is E1 BA BF, 3 bytes), "Tiếng Việt" is 14, + // "ệ" is 3, "Xin chào" is 9 (à is 2 bytes). + for (text, bytes) in [("Tiếng", 7), ("Tiếng Việt", 14), ("ệ", 3), ("Xin chào", 9)] { + let words = text.split_whitespace().count(); + // Fast path (`-m` only). + new_ucmd!() + .arg("-m") + .env("LC_ALL", "C") + .pipe_in(text) + .succeeds() + .stdout_is(format!("{bytes}\n")); + // Decoder path (`-wm`): same char count plus word count. + new_ucmd!() + .arg("-wm") + .env("LC_ALL", "C") + .pipe_in(text) + .succeeds() + .stdout_is(format!("{words:>7} {bytes:>7}\n")); + } + // POSIX names the same single-byte locale. + new_ucmd!() + .arg("-m") + .env("LC_ALL", "POSIX") + .pipe_in("Tiếng") + .succeeds() + .stdout_is("7\n"); +} + +#[rstest] +#[case::tieng("Tiếng", 5, 7, 1)] +#[case::tieng_viet("Tiếng Việt", 10, 14, 2)] +#[case::single_multibyte("ệ", 1, 3, 1)] +#[case::mixed_ascii("Xin chào", 8, 9, 2)] +fn test_wc_chars_utf8_locale( + #[case] text: &str, + #[case] chars: usize, + #[case] bytes: usize, + #[case] words: usize, +) { + // In UTF-8 locale, wc -m counts Unicode characters, not bytes. + // Byte counts: ế is E1 BA BF and ệ is E1 BB 87 (3 bytes each), à is 2 bytes. + // Fast path (`-m` only). + new_ucmd!() + .arg("-m") + .env("LC_ALL", "en_US.UTF-8") + .pipe_in(text) + .succeeds() + .stdout_is(format!("{chars}\n")); + // Decoder path (`-cmw`): same char count plus word and byte counts. + new_ucmd!() + .args(&["-cmw"]) + .env("LC_ALL", "en_US.UTF-8") + .pipe_in(text) + .succeeds() + .stdout_is(format!("{words:>7} {chars:>7} {bytes:>7}\n")); +} + +#[test] +fn test_wc_chars_default_locale() { + // With no locale configured, POSIX defaults to C. + let vietnamese_text = "Tiếng"; + + new_ucmd!() + .arg("-m") + .env("LC_ALL", "") + .env("LC_CTYPE", "") + .env("LANG", "") + .pipe_in(vietnamese_text) + .succeeds() + .stdout_is("7\n"); + + // Test with -w to trigger wc.rs path (word_count_from_reader_specialized) + // Order: words, chars + new_ucmd!() + .args(&["-mw"]) + .env("LC_ALL", "") + .env("LC_CTYPE", "") + .env("LANG", "") + .pipe_in(vietnamese_text) + .succeeds() + .stdout_is(" 1 7\n"); +} + +#[test] +fn test_char_count_invalid_bytes_locale() { + // `\xC3\xA9` (2 bytes / 1 UTF-8 char, e-acute) + `0xFF` (invalid byte). + let input = vec![0xC3, 0xA9, 0xFF]; + // Fast path (`-m` only) under C locale counts each byte. + new_ucmd!() + .arg("-m") + .env("LC_ALL", "C") + .pipe_in(input.clone()) + .succeeds() + .stdout_is("3\n"); + // NOTE: `-m` (fast path) with invalid bytes in a UTF-8 locale is + // intentionally unpinned: `bytecount` counts invalid lead bytes while + // GNU ignores encoding errors; fixing that is out of scope. + // Decoder path (`-wm`): GNU gives `1 3` under C and `1 1` under a UTF-8 + // locale (encoding errors are not counted in a multibyte locale). + new_ucmd!() + .arg("-wm") + .env("LC_ALL", "C") + .pipe_in(input.clone()) + .succeeds() + .stdout_is(" 1 3\n"); + new_ucmd!() + .arg("-wm") + .env("LC_ALL", "en_US.UTF-8") + .pipe_in(input) + .succeeds() + .stdout_is(" 1 1\n"); + // Invalid byte mid-buffer with word counting under C locale. + new_ucmd!() + .arg("-wm") + .env("LC_ALL", "C") + .pipe_in([b'a', 0xFF, b'b']) + .succeeds() + .stdout_is(" 1 3\n"); +}