diff --git a/crates/basal-host/src/builtins/git/windows.rs b/crates/basal-host/src/builtins/git/windows.rs index 81360377..9bf52d7c 100644 --- a/crates/basal-host/src/builtins/git/windows.rs +++ b/crates/basal-host/src/builtins/git/windows.rs @@ -964,6 +964,7 @@ mod spawn_lock_tests { use windows_sys::Win32::{ Foundation::{ CompareObjectHandles, DUPLICATE_SAME_ACCESS, DuplicateHandle, ERROR_INVALID_HANDLE, + ERROR_NOT_SUPPORTED, }, System::Threading::GetCurrentProcess, }; @@ -1134,16 +1135,139 @@ fn main() { ) } == 0 { - assert_eq!( - std::io::Error::last_os_error().raw_os_error(), - Some(ERROR_INVALID_HANDLE as i32), - "could not inspect a child's handle table" + let error = std::io::Error::last_os_error().raw_os_error(); + let kind = if error == Some(ERROR_NOT_SUPPORTED as i32) { + remote_object_type(process, value) + } else { + String::new() + }; + assert!( + unrelated_handle(error, &kind), + "could not inspect a child's handle table: value={value:#x} error={error:?} type={kind}" ); return None; } Some(OwnedHandle(duplicate)) } + fn unrelated_handle(error: Option, kind: &str) -> bool { + // Microsoft documents DuplicateHandle support for File and Pipe. + // The live EventRegister probe below observes ERROR_NOT_SUPPORTED for + // EtwRegistration, a separate object type, on both Windows CI images. + // Exempt only that type, never a File (which can be a pipe end). + error == Some(ERROR_INVALID_HANDLE as i32) + || (error == Some(ERROR_NOT_SUPPORTED as i32) && kind == "EtwRegistration") + } + + #[test] + fn windows_only_etw_registration_may_refuse_duplication() { + let unsupported = Some(ERROR_NOT_SUPPORTED as i32); + assert!(unrelated_handle(unsupported, "EtwRegistration")); + for kind in ["File", "Pipe", "Event", "", "unknown type"] { + assert!(!unrelated_handle(unsupported, kind), "{kind} was exempted"); + } + assert!(!unrelated_handle(Some(5), "EtwRegistration")); + assert!(!unrelated_handle(None, "EtwRegistration")); + assert!(unrelated_handle(Some(ERROR_INVALID_HANDLE as i32), "")); + } + + #[test] + fn windows_pipe_objects_remain_inspectable() { + let attributes = SECURITY_ATTRIBUTES { + nLength: std::mem::size_of::() as u32, + lpSecurityDescriptor: std::ptr::null_mut(), + bInheritHandle: 0, + }; + let (reader, writer) = pipe(&attributes).unwrap(); + let process = unsafe { GetCurrentProcess() }; + let value = reader.0 as usize; + assert_eq!(remote_object_type(process, value), "File"); + assert!(holds(process, value, &reader)); + assert!(!holds(process, writer.0 as usize, &reader)); + assert!(object_at(process, 0).is_none()); + } + + /// Resolve a remote handle's type without duplicating that handle. The + /// process snapshot contains a type index; NtQueryObject supplies its name. + fn remote_object_type(process: HANDLE, value: usize) -> String { + use std::ffi::c_void; + #[link(name = "ntdll", kind = "raw-dylib")] + unsafe extern "system" { + fn NtQueryInformationProcess( + process: HANDLE, + class: u32, + buffer: *mut c_void, + length: u32, + returned: *mut u32, + ) -> i32; + fn NtQueryObject( + handle: HANDLE, + class: u32, + buffer: *mut c_void, + length: u32, + returned: *mut u32, + ) -> i32; + } + #[repr(C)] + struct UnicodeString { + length: u16, + maximum_length: u16, + buffer: *const u16, + } + fn query(run: impl Fn(*mut c_void, u32, *mut u32) -> i32) -> Vec { + let mut bytes = 64 * 1024u32; + for _ in 0..8 { + let mut buffer = vec![0usize; bytes as usize / 8]; + let mut returned = 0; + let status = run(buffer.as_mut_ptr().cast(), bytes, &mut returned); + if matches!(status as u32, 0xc000_0004 | 0x8000_0005 | 0xc000_0023) { + bytes = returned.max(bytes).saturating_mul(2); + continue; + } + assert!( + status >= 0, + "handle type query: NTSTATUS {:#010x}", + status as u32 + ); + assert!(returned <= bytes); + return buffer; + } + panic!("handle type query kept growing"); + } + // The x64 ProcessHandleInformation layout has a two-word header and + // five-word entries; ObjectTypesInformation has a count followed by + // pointer-aligned type records. + assert_eq!(std::mem::size_of::(), 8); + let table = query(|buffer, bytes, returned| unsafe { + NtQueryInformationProcess(process, 51, buffer, bytes, returned) + }); + let index = (0..table[0]) + .find_map(|i| { + let entry = &table[2 + i * 5..2 + i * 5 + 5]; + (entry[0] == value).then_some((entry[3] >> 32) as u8) + }) + .expect("the unduplicable handle is still in the live child's table"); + let types = query(|buffer, bytes, returned| unsafe { + NtQueryObject(std::ptr::null_mut(), 3, buffer, bytes, returned) + }); + let base = types.as_ptr().cast::(); + let mut offset = 8; + for _ in 0..types[0] as u32 { + assert!(offset + 104 <= types.len() * 8); + let name = unsafe { &*base.add(offset).cast::() }; + if unsafe { *base.add(offset + 90) } == index { + let start = name.buffer as usize; + let end = start.checked_add(usize::from(name.length)).unwrap(); + assert!(start >= base as usize && end <= base as usize + types.len() * 8); + return String::from_utf16_lossy(unsafe { + std::slice::from_raw_parts(name.buffer, usize::from(name.length) / 2) + }); + } + offset = (offset + 104 + usize::from(name.maximum_length)).next_multiple_of(8); + } + panic!("unknown object type index {index}"); + } + /// Whether `process` holds `object` at `value`. A handle a child /// inherited keeps its value there, so another child's pipe end would /// sit at the value it has in its owner; comparing objects, not values, @@ -1153,6 +1277,107 @@ fn main() { .is_some_and(|found| unsafe { CompareObjectHandles(found.0, object.0) } != 0) } + /// Register a provider so the handle-type diagnostic does not depend on + /// another child's stdio value colliding with an ETW registration handle. + #[test] + fn windows_etw_registration_duplication_probe() { + use std::ffi::c_void; + #[link(name = "advapi32")] + unsafe extern "system" { + fn EventRegister( + provider: *const windows_sys::core::GUID, + callback: *const c_void, + context: *const c_void, + registration: *mut u64, + ) -> u32; + fn EventUnregister(registration: u64) -> u32; + } + #[link(name = "ntdll", kind = "raw-dylib")] + unsafe extern "system" { + fn NtQueryInformationProcess( + process: HANDLE, + class: u32, + buffer: *mut c_void, + length: u32, + returned: *mut u32, + ) -> i32; + } + struct Registration(u64); + impl Drop for Registration { + fn drop(&mut self) { + assert_eq!(unsafe { EventUnregister(self.0) }, 0); + } + } + let provider = windows_sys::core::GUID { + data1: 0xd9ca6e2b, + data2: 0xd6ad, + data3: 0x4658, + data4: [0x95, 0xc8, 0x0d, 0xca, 0x74, 0xe8, 0x21, 0xe4], + }; + let mut registration = Registration(0); + assert_eq!( + unsafe { + EventRegister( + &provider, + std::ptr::null(), + std::ptr::null(), + &mut registration.0, + ) + }, + 0 + ); + let process = unsafe { GetCurrentProcess() }; + let mut table = vec![0usize; 64 * 1024]; + let mut returned = 0; + let status = unsafe { + NtQueryInformationProcess( + process, + 51, + table.as_mut_ptr().cast(), + (table.len() * 8) as u32, + &mut returned, + ) + }; + assert!(status >= 0, "handle snapshot: {status:#x}"); + let mut unsupported = 0; + for i in 0..table[0] { + let value = table[2 + i * 5]; + let mut duplicate = std::ptr::null_mut(); + let ok = unsafe { + DuplicateHandle( + process, + value as HANDLE, + process, + &mut duplicate, + 0, + 0, + DUPLICATE_SAME_ACCESS, + ) + }; + if ok == 0 { + let error = std::io::Error::last_os_error().raw_os_error(); + if error != Some(ERROR_INVALID_HANDLE as i32) { + let name = remote_object_type(process, value); + println!( + "registered ETW provider: value={value:#x} error={error:?} type={name}" + ); + assert!( + unrelated_handle(error, &name), + "error={error:?} type={name}" + ); + assert!(object_at(process, value).is_none()); + unsupported += 1; + } + } else { + drop(OwnedHandle(duplicate)); + } + } + assert!( + unsupported > 0, + "ETW registration did not produce an unduplicable handle" + ); + } + /// Many confined launches, plain spawns and git children started at /// once: none of them holds another child's standard handle. A child /// that held one could read or write that child's pipe, and would keep diff --git a/crates/basal-worker/src/clock.rs b/crates/basal-worker/src/clock.rs index 775454d8..0cf9787e 100644 --- a/crates/basal-worker/src/clock.rs +++ b/crates/basal-worker/src/clock.rs @@ -125,22 +125,101 @@ mod tests { } } - /// Against the real clock. The budget and the burns span several - /// scheduler ticks (about 15.6 ms on Windows, where the thread CPU clock - /// advances a tick at a time), so tick rounding cannot flip the outcome. - #[test] - fn time_outside_the_engine_is_not_counted() { + #[cfg(windows)] + fn native_sample() -> String { + use windows_sys::Win32::{ + Foundation::FILETIME, + System::Threading::{GetCurrentThread, GetCurrentThreadId, GetThreadTimes}, + }; + #[link(name = "kernel32")] + unsafe extern "system" { + fn QueryPerformanceCounter(counter: *mut i64) -> i32; + fn QueryPerformanceFrequency(frequency: *mut i64) -> i32; + fn QueryThreadCycleTime(thread: *mut std::ffi::c_void, cycles: *mut u64) -> i32; + } + let zero = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let (mut creation, mut exit, mut kernel, mut user) = (zero, zero, zero, zero); + let (mut qpc, mut frequency, mut cycles) = (0, 0, 0); + // SAFETY: all output pointers are live and the pseudo-handle names this + // test thread. These independent counters are diagnostic, not a clock + // conversion or part of the engine's measured interval. + unsafe { + assert_ne!( + GetThreadTimes( + GetCurrentThread(), + &mut creation, + &mut exit, + &mut kernel, + &mut user + ), + 0, + "{}", + std::io::Error::last_os_error() + ); + assert_ne!(QueryPerformanceCounter(&mut qpc), 0); + assert_ne!(QueryPerformanceFrequency(&mut frequency), 0); + assert_ne!(QueryThreadCycleTime(GetCurrentThread(), &mut cycles), 0); + } + let units = |v: FILETIME| (u64::from(v.dwHighDateTime) << 32) | u64::from(v.dwLowDateTime); + format!( + "thread={} kernel_100ns={} user_100ns={} qpc={qpc} qpc_hz={frequency} cycles={cycles}", + unsafe { GetCurrentThreadId() }, + units(kernel), + units(user) + ) + } + + #[cfg(not(windows))] + fn native_sample() -> String { + format!("thread_cpu_time={:?}", thread_cpu_time()) + } + + fn engine_intervals(sample: usize) -> Duration { + let before = native_sample(); let clock = JsClock::new(Duration::from_millis(150)); clock.enter(); burn(Duration::from_millis(30)); clock.leave(); + let after = native_sample(); + let spent = clock.spent(); + let details = format!("sample={sample} spent={spent:?} before=[{before}] after=[{after}]"); + #[cfg(windows)] + println!("{details}"); // Sleeping and burning CPU while left must not count. std::thread::sleep(Duration::from_millis(50)); burn(Duration::from_millis(200)); - assert!(!clock.over_budget(), "spent {:?}", clock.spent()); + assert_eq!(clock.spent(), spent, "paused CPU time changed: {details}"); + assert!(!clock.over_budget(), "{details}"); clock.enter(); burn(Duration::from_millis(200)); - assert!(clock.over_budget()); + assert!( + clock.over_budget(), + "entered CPU time was not charged: {details}" + ); + spent + } + + /// Exercise the real CPU source as well as the pause/resume bookkeeping. + /// Keep the upper-budget check: an oversized charge can also end a real + /// activation early, even if pausing itself preserves the recorded spend. + #[test] + fn time_outside_the_engine_is_not_counted() { + engine_intervals(0); + } + + /// Compare the tick-accounted clock with wall time and raw thread cycles. + /// Keep every sample so a large accounting jump is visible in CI output. + #[cfg(windows)] + #[test] + fn windows_engine_interval_samples() { + let mut largest = Duration::ZERO; + for index in 0..512 { + largest = largest.max(engine_intervals(index)); + } + println!("largest entered interval over 512 samples: {largest:?}"); } thread_local! { diff --git a/mutations.d/host-builtins.toml b/mutations.d/host-builtins.toml index 6bdf6fe5..4ee62dda 100644 --- a/mutations.d/host-builtins.toml +++ b/mutations.d/host-builtins.toml @@ -629,6 +629,40 @@ file = "crates/basal-host/src/builtins/git/windows.rs" old = " let mut process: PROCESS_INFORMATION = unsafe { std::mem::zeroed() };\n // SAFETY: buffers, owned attribute payloads, handles and startup structure\n // remain live through creation; lpApplicationName is the canonical image.\n if unsafe {\n CreateProcessW(\n application.as_ptr(),\n command_line.as_mut_ptr(),\n std::ptr::null(),\n std::ptr::null(),\n 1,\n EXTENDED_STARTUPINFO_PRESENT\n | CREATE_NO_WINDOW\n | CREATE_UNICODE_ENVIRONMENT\n | CREATE_SUSPENDED,\n env.as_ptr().cast(),\n cwd_pointer,\n &startup.StartupInfo,\n &mut process,\n )\n } == 0\n {\n return Err(native_error(\"start git\"));\n }\n #[cfg(test)]\n spawn_lock_tests::created();\n // The child now holds its own copies of its three standard handles, so\n // the parent closes its copies before releasing the spawn lock. That\n // also lets the pipes report end of file once git exits.\n drop(attributes);\n drop((stdin, stdout_writer, stderr_writer));\n drop(spawn_lock);" new = " drop(spawn_lock); // NON-VACUITY BREAK\n let mut process: PROCESS_INFORMATION = unsafe { std::mem::zeroed() };\n // SAFETY: buffers, owned attribute payloads, handles and startup structure\n // remain live through creation; lpApplicationName is the canonical image.\n if unsafe {\n CreateProcessW(\n application.as_ptr(),\n command_line.as_mut_ptr(),\n std::ptr::null(),\n std::ptr::null(),\n 1,\n EXTENDED_STARTUPINFO_PRESENT\n | CREATE_NO_WINDOW\n | CREATE_UNICODE_ENVIRONMENT\n | CREATE_SUSPENDED,\n env.as_ptr().cast(),\n cwd_pointer,\n &startup.StartupInfo,\n &mut process,\n )\n } == 0\n {\n return Err(native_error(\"start git\"));\n }\n #[cfg(test)]\n spawn_lock_tests::created();\n // The child now holds its own copies of its three standard handles, so\n // the parent closes its copies before releasing the spawn lock. That\n // also lets the pipes report end of file once git exits.\n drop(attributes);\n drop((stdin, stdout_writer, stderr_writer));\n" +[[control]] +id = "windows-handle-probe-exempts-every-nonduplicable-type" +guards = "an unsupported duplicate is ignored only for EtwRegistration, never File or Pipe" +file = "crates/basal-host/src/builtins/git/windows.rs" +old = '&& kind == "EtwRegistration"' +new = '&& { let _ = kind; true } /* NON-VACUITY BREAK */' +test_file = "crates/basal-host/src/builtins/git/windows.rs" +runner = "cargo" +package = "basal-host" +platforms = ["windows"] +target = "--lib" +select = { tests = ["builtins::git::windows::spawn_lock_tests::windows_only_etw_registration_may_refuse_duplication", "builtins::git::windows::spawn_lock_tests::windows_pipe_objects_remain_inspectable"] } +expect_red = ["builtins::git::windows::spawn_lock_tests::windows_only_etw_registration_may_refuse_duplication"] +only = true +build_timeout_s = 3600 +timeout_s = 600 + +[[control]] +id = "windows-handle-probe-rejects-live-etw-registration" +guards = "a live nonduplicable ETW registration is not mistaken for another child's pipe" +file = "crates/basal-host/src/builtins/git/windows.rs" +old = '|| (error == Some(ERROR_NOT_SUPPORTED as i32) && kind == "EtwRegistration")' +new = '|| (false /* NON-VACUITY BREAK */ && kind == "EtwRegistration")' +test_file = "crates/basal-host/src/builtins/git/windows.rs" +runner = "cargo" +package = "basal-host" +platforms = ["windows"] +target = "--lib" +select = { tests = ["builtins::git::windows::spawn_lock_tests::windows_etw_registration_duplication_probe", "builtins::git::windows::spawn_lock_tests::windows_pipe_objects_remain_inspectable"] } +expect_red = ["builtins::git::windows::spawn_lock_tests::windows_etw_registration_duplication_probe"] +only = true +build_timeout_s = 3600 +timeout_s = 600 + # Windows filesystem path spelling, containment and replacement permissions; # Git config/hook isolation, output bounds and child-tree termination. diff --git a/mutations.d/worker.toml b/mutations.d/worker.toml index ac5292d1..61bb108e 100644 --- a/mutations.d/worker.toml +++ b/mutations.d/worker.toml @@ -359,6 +359,23 @@ edits = [ { file = "crates/basal-worker/src/clock.rs", old = " pub fn leave(&self) {\n", new = " pub fn leave(&self) {\n #[allow(unreachable_code)]\n return;\n" }, ] +[[control]] +id = "windows-js-clock-keeps-counting-after-leave" +guards = "CPU work outside the engine leaves its spent time unchanged on Windows" +file = "crates/basal-worker/src/clock.rs" +old = "if let Some(start) = self.entered_at.take() {" +new = "if let Some(start) = self.entered_at.get() { // NON-VACUITY BREAK" +test_file = "crates/basal-worker/src/clock.rs" +runner = "cargo" +package = "basal-worker" +platforms = ["windows"] +target = "--lib" +select = { tests = ["clock::tests::time_outside_the_engine_is_not_counted", "clock::tests::a_sample_below_the_entry_sample_counts_as_nothing"] } +expect_red = ["clock::tests::time_outside_the_engine_is_not_counted"] +only = true +build_timeout_s = 3600 +timeout_s = 600 + [[control]] id = "memory-limit-not-applied" hub = "Each activation's allocations are capped by its memory budget, asserted by the budget test, the allocator unit test and the top-level wrapper budget test" diff --git a/script/tests/witness_mutations.py b/script/tests/witness_mutations.py index 00092603..bcff066d 100644 --- a/script/tests/witness_mutations.py +++ b/script/tests/witness_mutations.py @@ -1,5 +1,7 @@ """Mutation witnesses must observe the indexed mutant and a named test failure.""" +import contextlib import importlib.util +import io from pathlib import Path import subprocess import tempfile @@ -40,7 +42,85 @@ def cargo(self, args, **kwargs): def test_feature_flags_apply_to_the_named_cargo_oracle(self): self.assertEqual(WITNESS.command(self.row), ["cargo", "test", "--locked", "-p", "fixture", - "--lib", "--features", "deviations", "--", "tests::guard", "--exact", "--nocapture"]) + "--lib", "--features", "deviations", "--", "tests::guard", "--exact", "--show-output"]) + + def cargo_with_sibling(self, args, **kwargs): + result = self.cargo(args, **kwargs) + if args[0] != "cargo": + return result + text = "test tests::sibling ... ok\n" + result.stdout + text = text.replace("0 passed; 1 failed", "1 passed; 1 failed") + text = text.replace("1 passed; 0 failed", "2 passed; 0 failed") + return subprocess.CompletedProcess(args, result.returncode, text) + + def test_explicit_run_set_executes_and_records_the_passing_sibling(self): + self.row["select"] = {"tests": ["tests::guard", "tests::sibling"]} + self.row["only"] = True + console = io.StringIO() + def run(args, **kwargs): + if args[0] == "cargo": + self.assertEqual(args[-4:], ["tests::guard", "tests::sibling", "--exact", "--show-output"]) + if "NON-VACUITY BREAK" in self.source.read_text(): + self.assertIn("+// NON-VACUITY BREAK", console.getvalue()) + self.assertIn("1 file changed", console.getvalue()) + return self.cargo_with_sibling(args, **kwargs) + with contextlib.redirect_stdout(console), mock.patch.object(WITNESS.subprocess, "run", side_effect=run): + proof = WITNESS.witness(self.row, self.output) + self.assertEqual(proof["outcome"], "reddened") + self.assertIn("tests::guard: FAILED", proof["captured_output"]) + self.assertIn("tests::sibling: ok", proof["captured_output"]) + self.assertIn("test tests::sibling ... ok", (self.output / "guard-baseline.log").read_text()) + self.assertIn("+// NON-VACUITY BREAK", (self.output / "guard-mutant.log").read_text()) + self.assertIn("+// NON-VACUITY BREAK", (self.output / "guard-applied.diff").read_text()) + self.assertIn("after: (empty)", (self.output / "guard-stats.log").read_text()) + self.assertEqual(self.source.read_text(), "guard();\n") + + def test_filter_run_set_runs_without_exact_and_records_its_sibling(self): + self.row["select"] = {"filter": "tests::"} + self.assertEqual(WITNESS.command(self.row)[-3:], ["--", "tests::", "--show-output"]) + with mock.patch.object(WITNESS.subprocess, "run", side_effect=self.cargo_with_sibling): + proof = WITNESS.witness(self.row, self.output) + self.assertEqual(proof["outcome"], "reddened") + self.assertIn("tests::sibling: ok", proof["captured_output"]) + + def test_captured_test_output_is_not_a_libtest_status(self): + for heading in ["successes", "failures"]: + with self.subTest(heading=heading): + text = (f"test tests::guard ... ok\n\n{heading}:\n" + "test tests::guard ... FAILED\ntest tests::sibling ... ok\n") + self.assertEqual(WITNESS.test_results(text), {"tests::guard": "ok"}) + + def test_a_missing_declared_sibling_is_refused_in_the_baseline(self): + self.row["select"] = {"tests": ["tests::guard", "tests::sibling"]} + with mock.patch.object(WITNESS.subprocess, "run", side_effect=self.cargo): + with self.assertRaisesRegex(SystemExit, "named baseline did not execute"): + WITNESS.witness(self.row, self.output) + self.assertEqual(self.source.read_text(), "guard();\n") + + def test_a_missing_sibling_in_the_mutant_is_not_a_complete_catch(self): + self.row["select"] = {"tests": ["tests::guard", "tests::sibling"]} + def run(args, **kwargs): + if args[0] == "cargo" and "NON-VACUITY BREAK" in self.source.read_text(): + return self.cargo(args, **kwargs) + return self.cargo_with_sibling(args, **kwargs) + with mock.patch.object(WITNESS.subprocess, "run", side_effect=run): + proof = WITNESS.witness(self.row, self.output) + self.assertEqual(proof["outcome"], "undefended") + self.assertEqual(self.source.read_text(), "guard();\n") + + def test_a_failing_sibling_is_not_a_named_mutation_catch(self): + self.row["select"] = {"tests": ["tests::guard", "tests::sibling"]} + self.row["only"] = True + def run(args, **kwargs): + result = self.cargo_with_sibling(args, **kwargs) + if args[0] == "cargo" and "NON-VACUITY BREAK" in self.source.read_text(): + return subprocess.CompletedProcess(args, 101, result.stdout.replace( + "test tests::sibling ... ok", "test tests::sibling ... FAILED")) + return result + with mock.patch.object(WITNESS.subprocess, "run", side_effect=run): + proof = WITNESS.witness(self.row, self.output) + self.assertEqual(proof["outcome"], "undefended") + self.assertEqual(self.source.read_text(), "guard();\n") def test_witness_stages_live_source_and_restores_it_after_a_named_failure(self): self.source.write_text("guard();\n// live implementation\n") diff --git a/script/witness-mutations.py b/script/witness-mutations.py index 4b47d99d..226dabc5 100644 --- a/script/witness-mutations.py +++ b/script/witness-mutations.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 -"""Capture indexed source diffs and exact named-test failures for Cargo controls. +"""Capture indexed source diffs and named-test run sets for Cargo controls. -The pinned ckdev-mutate runner grades whether catalogue mutations are caught by -their declared test targets. This script adds the applied/restored Git diff and -an exact named-test failure; it does not replace that replay or coverage grading. +The pinned ckdev-mutate runner replays catalogue mutations against their declared +test targets. This script supplements that replay with applied/restored Git diffs, +a named failure and any declared passing siblings. It does not replace the runner's +catch or coverage grading. Rows without a run set use only their named oracle. """ import argparse import json from pathlib import Path +import re import shlex import subprocess import sys @@ -30,7 +32,33 @@ def command(row): args.append("--no-default-features") if row.get("all_features"): args.append("--all-features") - return args + ["--", row["expect_red"][0], "--exact", "--nocapture"] + selection = row.get("select") + if isinstance(selection, dict) and "tests" in selection: + filters = [*selection["tests"], "--exact"] + elif isinstance(selection, dict) and "filter" in selection: + filters = [selection["filter"]] + else: + filters = [row["expect_red"][0], "--exact"] + # Keep test output captured until libtest finishes its status lines. Live + # output can interleave with those lines and make the results unparseable. + return args + ["--", *filters, "--show-output"] + + +def test_results(text): + # With --show-output, captured test text follows these headings. Do not + # mistake a test's own output for libtest's status lines. + text = re.split(r"^(?:failures|successes):\s*$", text, maxsplit=1, flags=re.MULTILINE)[0] + return dict(re.findall(r"^test (\S+) \.\.\. (ok|FAILED|ignored)\b", text, re.MULTILINE)) + + +def complete_selection(row, results): + selection = row.get("select") + if isinstance(selection, dict) and "tests" in selection: + return set(results) == set(selection["tests"]) + if isinstance(selection, dict) and "filter" in selection: + return (len(results) >= 2 and row["expect_red"][0] in results + and all(selection["filter"] in name for name in results)) + return set(results) == {row["expect_red"][0]} def witness(row, output): @@ -46,9 +74,12 @@ def witness(row, output): args = command(row) baseline = subprocess.run(args, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=row.get("build_timeout_s", 3600)) + (output / f"{row['id']}-baseline.log").write_text(baseline.stdout) test = row["expect_red"][0] - if baseline.returncode or f"test {test} ... ok" not in baseline.stdout: - (output / f"{row['id']}-baseline.log").write_text(baseline.stdout) + baseline_results = test_results(baseline.stdout) + if (baseline.returncode or baseline_results.get(test) != "ok" + or not complete_selection(row, baseline_results) + or any(status != "ok" for status in baseline_results.values())): raise SystemExit(f"{row['id']}: named baseline did not execute and pass") result = None try: @@ -58,6 +89,10 @@ def witness(row, output): during = git("diff", "--stat") if not during: raise SystemExit("mutation safety: empty applied diff") + applied = git("diff", "--", row["file"]) + (output / f"{row['id']}-applied.diff").write_text(applied + "\n") + prefix = f"Applied mutation {row['id']}:\n{applied}\nDuring diff stat:\n{during}\n" + print(prefix, flush=True) try: result = subprocess.run(args, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=row.get("build_timeout_s", 3600)) @@ -70,14 +105,20 @@ def witness(row, output): after = git("diff", "--stat") if after: raise SystemExit("mutation safety: restored diff is not empty") - (output / f"{row['id']}-mutant.log").write_text(text) - expected = f"test {test} ... FAILED" - red = result is not None and result.returncode != 0 and expected in text and "0 passed; 1 failed" in text - reached = f"test {test} ..." in text + stats = f"during: {during}\nafter: {after or '(empty)'}\n" + (output / f"{row['id']}-stats.log").write_text(stats) + print(f"Restored {row['file']}: {after or '(empty)'}", flush=True) + (output / f"{row['id']}-mutant.log").write_text(prefix + text) + results = test_results(text) + red = (result is not None and result.returncode != 0 and results.get(test) == "FAILED" + and complete_selection(row, results) + and all(status == "ok" for name, status in results.items() if name != test)) + reached = test in results + summary = "; ".join(f"{name}: {status}" for name, status in sorted(results.items())) proof = { "control": row["id"], "expected_red": test, - "captured_output": expected + "; 0 passed; 1 failed; all other tests filtered" if red else text[-400:], + "captured_output": summary if red else text[-400:], "applied_evidence": f"{row['file']}; during: {during}; after: {after or '(empty)'}", "outcome": "reddened" if red else "hung" if result is None else "not_reached" if not reached else "undefended", }