Fix UTF-16 requirements.txt silently skipped (#721)

crates/socket-patch-cli/src/commands/scan/hosted.rs

@@ -1862,6 +1862,26 @@ async fn vendored_takeover(
                     .filter(|_| entry.is_some_and(vlt_entry))
             })
     };
+    // NON-UTF-8 PRE-CHECK (#721) — the GUARD's undecodable-file rule
+    // (`engine::undecodable_guard`), checked BEFORE any revert dispatches
+    // (and under --dry-run too): a takeover that reverted first and was
+    // then refused by the guard would leave the reverted purls unpatched
+    // in both modes.
+    if takeover.iter().any(|(_, entry)| entry.is_some()) {
+        let view = socket_patch_core::vendor::lock_inventory::ProjectView::Disk(&common.cwd);
+        let read = socket_patch_core::hosted::engine::read_candidate_files(
+            &view,
+            &std::collections::BTreeSet::new(),
+            candidates,
+        )
+        .await;
+        if let Some(refusal) = socket_patch_core::hosted::engine::undecodable_guard(
+            &read.undecodable_reads,
+            candidates,
+        ) {
+            return Err(refusal);
+        }
+    }
     // SYMLINK PRE-CHECK for the takeover reverts — the same rule as the
     // SYMLINK GUARD below, applied to each ledger entry's recorded wiring
     // (the revert backends also stage and rename over the file). Checked

crates/socket-patch-cli/tests/in_process_get_hosted_ecosystems.rs

@@ -315,6 +315,61 @@ async fn pypi_requirements_hosted_rewrites_pep440_equivalent_pin() {
     }
 }
 
+/// #721: Windows PowerShell 5.1 writes `pip freeze > requirements.txt` as
+/// UTF-16 with a BOM, and pip installs from it. The hosted grant must not
+/// treat that file as absent and exit 0 with the project unpatched: it is
+/// refused by name (`candidate_file_unreadable`, exit 1), nothing written.
+#[tokio::test]
+#[serial]
+async fn pypi_requirements_hosted_refuses_a_utf16_file() {
+    const UUID: &str = "a1a1a1a1-a1a1-4a1a-8a1a-a1a1a1a1a1a3";
+    const PURL: &str = "pkg:pypi/requests@2.31.0";
+    const SHA256: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+    let url = format!(
+        "http://patch.test/patch/pypi/requests/2.31.0/{TOKEN}/{UUID}/requests-2.31.0-py3-none-any.whl"
+    );
+
+    let text = "flask==2.0.1\r\nrequests==2.31.0\r\n";
+    let le: Vec<u8> = [0xFF, 0xFE]
+        .into_iter()
+        .chain(text.encode_utf16().flat_map(u16::to_le_bytes))
+        .collect();
+    let be: Vec<u8> = [0xFE, 0xFF]
+        .into_iter()
+        .chain(text.encode_utf16().flat_map(u16::to_be_bytes))
+        .collect();
+    for (what, bytes) in [("utf-16le", le), ("utf-16be", be)] {
+        let server = MockServer::start().await;
+        mock_view(&server, UUID, PURL).await;
+        mock_reference(
+            &server,
+            UUID,
+            PURL,
+            &url,
+            serde_json::json!({ "sha256": SHA256 }),
+            serde_json::Value::Null,
+        )
+        .await;
+
+        let tmp = tempfile::tempdir().unwrap();
+        std::fs::write(tmp.path().join("requirements.txt"), &bytes).unwrap();
+
+        let code =
+            socket_patch_cli::commands::get::run(get_hosted_args(UUID, tmp.path(), server.uri()))
+                .await;
+        assert_eq!(
+            code, 1,
+            "{what}: a requirements.txt hosted mode cannot read must refuse, not exit 0 unpatched"
+        );
+        assert_eq!(
+            std::fs::read(tmp.path().join("requirements.txt")).unwrap(),
+            bytes,
+            "{what}: the refused file must stay byte-identical"
+        );
+        assert_no_manifest_no_blobs(tmp.path());
+    }
+}
+
 // ---------------------------------------------------------------------------
 // maven — pom.xml fail-closed suffixed-version pin (rewrite_maven_pom)
 // ---------------------------------------------------------------------------

crates/socket-patch-cli/tests/mode_migration_pypi.rs

@@ -564,6 +564,64 @@ async fn uv_takeover_without_wheel_metadata_fails_loudly() {
 /// the revert (the artifact and ledger entry are kept). The takeover must
 /// then refuse — keeping the ledger — rather than drop the entry and leave
 /// the project half vendored with no record of it.
+/// #721: a non-UTF-8 candidate file (here a UTF-16 `pip freeze` export
+/// beside a vendored Poetry project) refuses the hosted run BEFORE the
+/// takeover reverts anything, wet and `--dry-run` alike: refusing only at
+/// the rewrite would leave the reverted poetry.lock unpatched in both modes.
+#[tokio::test]
+async fn undecodable_candidate_refuses_before_the_takeover_reverts() {
+    let (_tmp, root) = project();
+    std::fs::write(
+        root.join("pyproject.toml"),
+        "[tool.poetry]\nname = \"demo\"\nversion = \"0.1.0\"\ndescription = \"\"\nauthors = [\"x <x@x>\"]\npackage-mode = false\n\n[tool.poetry.dependencies]\npython = \">=3.9\"\nsix = \"1.16.0\"\n",
+    )
+    .unwrap();
+    std::fs::write(
+        root.join("poetry.lock"),
+        POETRY_LOCK
+            .replace("WHEEL_SHA", WHEEL_SHA)
+            .replace("SDIST_SHA", SDIST_SHA),
+    )
+    .unwrap();
+    vendor_project(&root, &["poetry.lock", "pyproject.toml"]);
+    let mut utf16 = vec![0xFF, 0xFE];
+    for unit in "six==1.16.0\r\n".encode_utf16() {
+        utf16.extend(unit.to_le_bytes());
+    }
+    std::fs::write(root.join("requirements.txt"), &utf16).unwrap();
+    let lock = std::fs::read_to_string(root.join("poetry.lock")).unwrap();
+    let state = root.join(".socket/vendor/state.json");
+
+    let server = MockServer::start().await;
+    mount_hosted_api(&server, true).await;
+    let uri = server.uri();
+    for dry_run in [true, false] {
+        let mut args = hosted_scan_args(&uri);
+        if dry_run {
+            args.push("--dry-run");
+        }
+        let (code, env) = run_cli(&root, &args, &[]);
+        assert_eq!(code, 1, "dry_run={dry_run}: {env:#}");
+        let text = env.to_string();
+        assert!(
+            text.contains("candidate_file_unreadable") && text.contains("requirements.txt"),
+            "dry_run={dry_run}: {env:#}"
+        );
+        assert!(
+            !text.contains("redirect_takeover_reverted_vendored"),
+            "dry_run={dry_run}: nothing is reverted: {env:#}"
+        );
+        assert_eq!(
+            std::fs::read_to_string(root.join("poetry.lock")).unwrap(),
+            lock,
+            "dry_run={dry_run}: the vendored lock is untouched"
+        );
+        assert!(std::fs::read_to_string(&state).unwrap().contains(UUID));
+        assert!(root.join(format!(".socket/vendor/pypi/{UUID}")).exists());
+        assert_eq!(std::fs::read(root.join("requirements.txt")).unwrap(), utf16);
+    }
+}
+
 #[tokio::test]
 async fn drifted_vendored_line_refuses_takeover() {
     let (_tmp, root) = project();

crates/socket-patch-cli/tests/scan_requirements_lock_only.rs

@@ -5,7 +5,9 @@
 //! release:
 //!
 //! * #523: whitespace around `==` and the legacy `name (==X)` form;
-//! * #412: pins reached through in-root `-r` includes.
+//! * #412: pins reached through in-root `-r` includes;
+//! * #721: a UTF-16 file with a BOM (Windows PowerShell 5.1's
+//!   `pip freeze >` output), which pip decodes.
 //!
 //! Driven through the built binary against a mock patch API; the
 //! assertion is what discovery sends to the batch endpoint and the
@@ -87,6 +89,11 @@ async fn batch_purls(mock: &MockServer) -> Vec<String> {
 }
 
 async fn assert_lock_only_discovers(files: &[(&str, &str)], expected: &[&str]) {
+    let files: Vec<(&str, &[u8])> = files.iter().map(|(r, c)| (*r, c.as_bytes())).collect();
+    assert_lock_only_discovers_bytes(&files, expected).await;
+}
+
+async fn assert_lock_only_discovers_bytes(files: &[(&str, &[u8])], expected: &[&str]) {
     for mode in [&[][..], &["--vendor"][..]] {
         let mock = MockServer::start().await;
         mount_empty_batch(&mock).await;
@@ -148,3 +155,29 @@ async fn lock_only_scan_discovers_included_pins() {
     )
     .await;
 }
+
+/// #721: pip decodes a requirements file by its BOM, so a UTF-16 file
+/// (what Windows PowerShell 5.1's `pip freeze >` writes) is discovered,
+/// in either byte order, instead of reading as "No packages found".
+#[tokio::test]
+async fn lock_only_scan_discovers_utf16_pins() {
+    let text = "sp-fixture-idna==3.7\r\nsp-fixture-six==1.16.0\r\n";
+    let le: Vec<u8> = [0xFF, 0xFE]
+        .into_iter()
+        .chain(text.encode_utf16().flat_map(u16::to_le_bytes))
+        .collect();
+    let be: Vec<u8> = [0xFE, 0xFF]
+        .into_iter()
+        .chain(text.encode_utf16().flat_map(u16::to_be_bytes))
+        .collect();
+    for bytes in [le, be] {
+        assert_lock_only_discovers_bytes(
+            &[("requirements.txt", &bytes)],
+            &[
+                "pkg:pypi/sp-fixture-idna@3.7",
+                "pkg:pypi/sp-fixture-six@1.16.0",
+            ],
+        )
+        .await;
+    }
+}

crates/socket-patch-core/src/crawlers/gradle_cache.rs

@@ -70,8 +70,7 @@ pub fn hash_eq(dir_name: &str, sha1_hex: &str) -> bool {
 /// Whether `bytes` are the pristine download Gradle stored in the hash
 /// directory `dir_name` (their sha1 names it).
 pub fn pristine(dir_name: &str, bytes: &[u8]) -> bool {
-    use sha1::{Digest, Sha1};
-    hash_eq(dir_name, &hex::encode(Sha1::digest(bytes)))
+    hash_eq(dir_name, &crate::utils::digest::sha1_hex_of(bytes))
 }
 
 /// Whether `path` is a version directory of a `files-2.1` tree
@@ -432,8 +431,6 @@ impl DerivedIndex {
     /// The [`DerivedCopies`] of the jar `jar_leaf` whose pristine bytes
     /// hash to `pristine_sha1`.
     pub fn query(&self, jar_leaf: &str, pristine_sha1: &str) -> DerivedCopies {
-        use sha1::{Digest, Sha1};
-
         let instrumented = format!("instrumented-{jar_leaf}");
         let mut out = DerivedCopies {
             incomplete: self.incomplete,
@@ -460,7 +457,9 @@ impl DerivedIndex {
                 out.stale.push(path.clone());
             } else if name == jar_leaf || name == instrumented {
                 match crate::utils::fs::read_regular_to_bytes_sync(path) {
-                    Ok(bytes) if hash_eq(&hex::encode(Sha1::digest(&bytes)), pristine_sha1) => {
+                    Ok(bytes)
+                        if hash_eq(&crate::utils::digest::sha1_hex_of(&bytes), pristine_sha1) =>
+                    {
                         out.stale.push(path.clone())
                     }
                     Ok(_) => out.unknown.push(path.clone()),