Speed up the remaining targeted harness tests

scripts/tests/_test_helpers.py

@@ -24,7 +24,7 @@
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
-from typing import Any, Callable, List, Optional, Sequence, Tuple
+from typing import Any, Callable, List, Mapping, Optional, Sequence, Tuple
 
 _failure_count_value = 0
 
@@ -200,7 +200,7 @@ def find_python() -> str:
     raise RuntimeError("the harness self-tests require an executable Python interpreter")
 
 
-def run_tool_output(command: Sequence[str]) -> Tuple[str, int]:
+def run_tool_output(command: Sequence[str], *, env: Optional[Mapping[str, str]] = None) -> Tuple[str, int]:
     """Run a child tool with all output merged, returning ``(output, exit code)``.
 
     This mirrors the PowerShell originals' ``*>&1 | Out-String``: stdout and
@@ -210,6 +210,7 @@ def run_tool_output(command: Sequence[str]) -> Tuple[str, int]:
     """
     completed = subprocess.run(
         list(command),
+        env=env,
         capture_output=True,
         text=True,
         encoding="utf-8",

scripts/tests/test_assembly_warnings.py

@@ -24,7 +24,7 @@
     find_python,
     new_temp_root,
     remove_temp_root,
-    run_case,
+    run_cases,
     write_fixture_file,
 )
 
@@ -308,36 +308,19 @@ def main() -> int:
     # interpreter because the fixtures never touch a toolchain.
     python = find_python()
 
-    run_case("Passes_ForRepositoryAssemblies", lambda: case_passes_for_repository_assemblies(python))
-    run_case(
-        "Passes_When_EveryAssemblyHasWarningsAsErrors",
-        lambda: case_passes_when_every_assembly_has_warnings_as_errors(python),
-    )
-    run_case("Fails_WhenAssemblyHasNoRuleset", lambda: case_fails_when_assembly_has_no_ruleset(python))
-    run_case(
-        "Fails_WhenAssemblyHasNoCompilerResponseFile",
-        lambda: case_fails_when_assembly_has_no_compiler_response_file(python),
-    )
-    run_case(
-        "Fails_WhenAssemblyHasAmbiguousCompilerResponseFiles",
-        lambda: case_fails_when_assembly_has_ambiguous_compiler_response_files(python),
-    )
-    for response_file_case in RESPONSE_FILE_CASES:
-        run_case(response_file_case.name, lambda case=response_file_case: run_response_file_case(python, case))
-    run_case("Fails_WhenAssemblyHasMultipleRulesets", lambda: case_fails_when_assembly_has_multiple_rulesets(python))
-    run_case("Fails_WhenRulesetIsMalformed", lambda: case_fails_when_ruleset_is_malformed(python))
-    run_case(
-        "Fails_WhenIncludeAllDoesNotPromoteWarnings",
-        lambda: case_fails_when_include_all_does_not_promote_warnings(python),
-    )
-    run_case(
-        "Fails_WhenRepositoryContainsLabeledAnalyzer",
-        lambda: case_fails_when_repository_contains_labeled_analyzer(python),
-    )
-    run_case(
-        "Fails_WhenRepositoryContainsKnownAnalyzerPayload",
-        lambda: case_fails_when_repository_contains_known_analyzer_payload(python),
-    )
+    run_cases([
+        ("Passes_ForRepositoryAssemblies", lambda: case_passes_for_repository_assemblies(python)),
+        ("Passes_When_EveryAssemblyHasWarningsAsErrors", lambda: case_passes_when_every_assembly_has_warnings_as_errors(python)),
+        ("Fails_WhenAssemblyHasNoRuleset", lambda: case_fails_when_assembly_has_no_ruleset(python)),
+        ("Fails_WhenAssemblyHasNoCompilerResponseFile", lambda: case_fails_when_assembly_has_no_compiler_response_file(python)),
+        ("Fails_WhenAssemblyHasAmbiguousCompilerResponseFiles", lambda: case_fails_when_assembly_has_ambiguous_compiler_response_files(python)),
+        *[(response_file_case.name, lambda case=response_file_case: run_response_file_case(python, case)) for response_file_case in RESPONSE_FILE_CASES],
+        ("Fails_WhenAssemblyHasMultipleRulesets", lambda: case_fails_when_assembly_has_multiple_rulesets(python)),
+        ("Fails_WhenRulesetIsMalformed", lambda: case_fails_when_ruleset_is_malformed(python)),
+        ("Fails_WhenIncludeAllDoesNotPromoteWarnings", lambda: case_fails_when_include_all_does_not_promote_warnings(python)),
+        ("Fails_WhenRepositoryContainsLabeledAnalyzer", lambda: case_fails_when_repository_contains_labeled_analyzer(python)),
+        ("Fails_WhenRepositoryContainsKnownAnalyzerPayload", lambda: case_fails_when_repository_contains_known_analyzer_payload(python)),
+    ])
 
     failures = failure_count()
     print(f"== assembly warnings policy: {failures} failure(s) ==")

scripts/tests/test_changelog_length.py

@@ -24,7 +24,7 @@
     find_python,
     new_temp_root,
     remove_temp_root,
-    run_case,
+    run_cases,
     run_tool_output,
     write_fixture_file,
 )
@@ -38,17 +38,10 @@
 
 
 def invoke_changelog_length_lint(changelog_path: Path, lint_arguments: Sequence[str] = ()) -> Tuple[str, int]:
-    """Run the lint with the changelog override env var set, then restore it."""
-    had_override = "CHANGELOG_LENGTH_CHANGELOG" in os.environ
-    previous_override = os.environ.get("CHANGELOG_LENGTH_CHANGELOG")
-    try:
-        os.environ["CHANGELOG_LENGTH_CHANGELOG"] = str(changelog_path)
-        return run_tool_output([find_python(), str(LINT_SCRIPT), *lint_arguments])
-    finally:
-        if had_override:
-            os.environ["CHANGELOG_LENGTH_CHANGELOG"] = previous_override or ""
-        else:
-            os.environ.pop("CHANGELOG_LENGTH_CHANGELOG", None)
+    """Run the lint with private child-environment overrides."""
+    env = dict(os.environ)
+    env["CHANGELOG_LENGTH_CHANGELOG"] = str(changelog_path)
+    return run_tool_output([find_python(), str(LINT_SCRIPT), *lint_arguments], env=env)
 
 
 def invoke_fixture_lint(root: Path, lint_arguments: Sequence[str] = ()) -> Tuple[str, int]:
@@ -319,12 +312,13 @@ def case_passes_against_the_repository_itself() -> None:
 def main() -> int:
     print("== Changelog entry length self-tests ==")
 
-    for case in CASES:
-        run_case(case.name, lambda bound_case=case: run_table_case(bound_case))
-    run_case("Fails_WhenUnreleasedIsMissing", case_fails_when_unreleased_is_missing)
-    run_case("Fails_WhenUnreleasedIsDuplicated", case_fails_when_unreleased_is_duplicated)
-    run_case("Fails_WhenTheChangelogIsMissing", case_fails_when_the_changelog_is_missing)
-    run_case("Passes_AgainstTheRepositoryItself", case_passes_against_the_repository_itself)
+    run_cases([
+        *[(case.name, lambda bound_case=case: run_table_case(bound_case)) for case in CASES],
+        ("Fails_WhenUnreleasedIsMissing", case_fails_when_unreleased_is_missing),
+        ("Fails_WhenUnreleasedIsDuplicated", case_fails_when_unreleased_is_duplicated),
+        ("Fails_WhenTheChangelogIsMissing", case_fails_when_the_changelog_is_missing),
+        ("Passes_AgainstTheRepositoryItself", case_passes_against_the_repository_itself),
+    ])
 
     failures = failure_count()
     if failures > 0:

scripts/tests/test_csharp_member_order.py

@@ -23,7 +23,7 @@
     fixture_path,
     new_temp_root,
     remove_temp_root,
-    run_case,
+    run_cases,
     run_tool_output,
     write_fixture_file,
 )
@@ -34,17 +34,10 @@
 
 
 def invoke_member_order_lint(root: Path, lint_arguments: Sequence[str] = ()) -> Tuple[str, int]:
-    """Run the lint with the roots override env var set, then restore it."""
-    had_previous = "CSHARP_MEMBER_ORDER_ROOTS" in os.environ
-    previous = os.environ.get("CSHARP_MEMBER_ORDER_ROOTS")
-    try:
-        os.environ["CSHARP_MEMBER_ORDER_ROOTS"] = str(root)
-        return run_tool_output([find_python(), str(LINT_SCRIPT), *lint_arguments])
-    finally:
-        if had_previous:
-            os.environ["CSHARP_MEMBER_ORDER_ROOTS"] = previous or ""
-        else:
-            os.environ.pop("CSHARP_MEMBER_ORDER_ROOTS", None)
+    """Run the lint with private child-environment overrides."""
+    env = dict(os.environ)
+    env["CSHARP_MEMBER_ORDER_ROOTS"] = str(root)
+    return run_tool_output([find_python(), str(LINT_SCRIPT), *lint_arguments], env=env)
 
 
 ORDERED_FIXTURE = """using System;
@@ -609,28 +602,27 @@ def case_paths_scans_only_the_named_files() -> None:
 def main() -> int:
     print("== C# member-order self-tests ==")
 
-    run_case("Passes_CanonicalOrder", case_passes_canonical_order)
-    run_case("Fails_OnTierAndAccessInversions", case_fails_on_tier_and_access_inversions)
-    run_case("Fails_OnUnderscoredMethodName", case_fails_on_underscored_method_name)
-    run_case("Fails_OnUnderscoredGenericMethodName", case_fails_on_underscored_generic_method_name)
-    run_case("Fails_OnConsecutiveOrdinaryLineComments", case_fails_on_consecutive_ordinary_line_comments)
-    run_case(
-        "Fails_OnLeadingBomConsecutiveOrdinaryLineComments",
-        case_fails_on_leading_bom_consecutive_ordinary_line_comments,
-    )
-    run_case("Passes_SupportedCommentForms", case_passes_supported_comment_forms)
-    run_case("Fails_OnRuntimeLinqImportsAndQualifiedCalls", case_fails_on_runtime_linq_imports_and_qualified_calls)
-    run_case("Passes_RuntimeLinqTextInCommentsAndStrings", case_passes_runtime_linq_text_in_comments_and_strings)
-    run_case("Fails_OnPersistedStateModelLinqDependency", case_fails_on_persisted_state_model_linq_dependency)
-    run_case("Fails_OnSearchModelLinqDependency", case_fails_on_search_model_linq_dependency)
-    run_case("Fails_OnAssetPostprocessingLinqDependency", case_fails_on_asset_postprocessing_linq_dependency)
-    run_case("Fails_OnNamespaceDiscoveryLinqDependency", case_fails_on_namespace_discovery_linq_dependency)
-    run_case("Passes_OtherEditorLinqDependency", case_passes_other_editor_linq_dependency)
-    run_case("Fails_WhenNestedTypeIsInterspersed", case_fails_when_nested_type_is_interspersed)
-    run_case("FixesTierOrderByExactPermutation", case_fixes_tier_order_by_exact_permutation)
-    run_case("Fails_OnCountedArrayLoopWithUnusedIndex", case_fails_on_counted_array_loop_with_unused_index)
-    run_case("Passes_CountedLoopsWhenTheIndexIsNeeded", case_passes_counted_loops_when_the_index_is_needed)
-    run_case("Paths_ScansOnlyTheNamedFiles", case_paths_scans_only_the_named_files)
+    run_cases([
+        ("Passes_CanonicalOrder", case_passes_canonical_order),
+        ("Fails_OnTierAndAccessInversions", case_fails_on_tier_and_access_inversions),
+        ("Fails_OnUnderscoredMethodName", case_fails_on_underscored_method_name),
+        ("Fails_OnUnderscoredGenericMethodName", case_fails_on_underscored_generic_method_name),
+        ("Fails_OnConsecutiveOrdinaryLineComments", case_fails_on_consecutive_ordinary_line_comments),
+        ("Fails_OnLeadingBomConsecutiveOrdinaryLineComments", case_fails_on_leading_bom_consecutive_ordinary_line_comments),
+        ("Passes_SupportedCommentForms", case_passes_supported_comment_forms),
+        ("Fails_OnRuntimeLinqImportsAndQualifiedCalls", case_fails_on_runtime_linq_imports_and_qualified_calls),
+        ("Passes_RuntimeLinqTextInCommentsAndStrings", case_passes_runtime_linq_text_in_comments_and_strings),
+        ("Fails_OnPersistedStateModelLinqDependency", case_fails_on_persisted_state_model_linq_dependency),
+        ("Fails_OnSearchModelLinqDependency", case_fails_on_search_model_linq_dependency),
+        ("Fails_OnAssetPostprocessingLinqDependency", case_fails_on_asset_postprocessing_linq_dependency),
+        ("Fails_OnNamespaceDiscoveryLinqDependency", case_fails_on_namespace_discovery_linq_dependency),
+        ("Passes_OtherEditorLinqDependency", case_passes_other_editor_linq_dependency),
+        ("Fails_WhenNestedTypeIsInterspersed", case_fails_when_nested_type_is_interspersed),
+        ("FixesTierOrderByExactPermutation", case_fixes_tier_order_by_exact_permutation),
+        ("Fails_OnCountedArrayLoopWithUnusedIndex", case_fails_on_counted_array_loop_with_unused_index),
+        ("Passes_CountedLoopsWhenTheIndexIsNeeded", case_passes_counted_loops_when_the_index_is_needed),
+        ("Paths_ScansOnlyTheNamedFiles", case_paths_scans_only_the_named_files),
+    ])
 
     if failure_count() > 0:
         return 1

scripts/tests/test_devcontainer_lifecycle.py

@@ -24,7 +24,7 @@
 
 sys.dont_write_bytecode = True
 
-from _test_helpers import assert_exit_code, assert_true, failure_count, run_case
+from _test_helpers import assert_exit_code, assert_true, failure_count, run_cases
 
 TESTS_DIR = Path(__file__).resolve().parent
 REPO_ROOT = TESTS_DIR.parent.parent
@@ -388,13 +388,15 @@ def main() -> int:
         print("== devcontainer lifecycle: 0 failure(s) ==")
         return 0
 
-    run_case("OnCreate_BootstrapsEnvLocalAndIsIdempotent", case_on_create_bootstraps_and_is_idempotent)
-    run_case("OnCreate_ToleratesNpmRefreshFailureOnly", case_on_create_tolerates_npm_failure_only)
-    run_case("PostCreate_RetriesUntilTheThirdAttemptSucceeds", case_post_create_retries_until_success)
-    run_case("PostCreate_InstallsPreCommitHooksBestEffort", case_post_create_installs_pre_commit_hooks)
-    run_case("PostStart_ReplacesBashrcBlockAndLoadsEnvWithFilePrecedence", case_post_start_replaces_block_and_loads_env)
-    run_case("UpdateContent_RestoresAndSyncsOnlyWhenPresent", case_update_content_restores_and_syncs_only_when_present)
-    run_case("Lifecycle_MissingWorkspaceFailsForEveryEntry", case_missing_workspace_fails_for_every_entry)
+    run_cases([
+        ("OnCreate_BootstrapsEnvLocalAndIsIdempotent", case_on_create_bootstraps_and_is_idempotent),
+        ("OnCreate_ToleratesNpmRefreshFailureOnly", case_on_create_tolerates_npm_failure_only),
+        ("PostCreate_RetriesUntilTheThirdAttemptSucceeds", case_post_create_retries_until_success),
+        ("PostCreate_InstallsPreCommitHooksBestEffort", case_post_create_installs_pre_commit_hooks),
+        ("PostStart_ReplacesBashrcBlockAndLoadsEnvWithFilePrecedence", case_post_start_replaces_block_and_loads_env),
+        ("UpdateContent_RestoresAndSyncsOnlyWhenPresent", case_update_content_restores_and_syncs_only_when_present),
+        ("Lifecycle_MissingWorkspaceFailsForEveryEntry", case_missing_workspace_fails_for_every_entry),
+    ])
 
     failures = failure_count()
     print(f"== devcontainer lifecycle: {failures} failure(s) ==")

scripts/tests/test_gate.py

@@ -26,7 +26,7 @@
     find_python,
     new_temp_root,
     remove_temp_root,
-    run_case,
+    run_cases,
     write_fixture_file,
 )
 
@@ -602,71 +602,28 @@ def main() -> int:
     # only need a working interpreter because --plan executes nothing.
     python = find_python()
 
-    run_case("Plan_PrintsTheFullLaneTable", lambda: case_plan_prints_the_full_lane_table(python))
-    run_case("Plan_CarriesCommandsAndDependencies", lambda: case_plan_carries_commands_and_dependencies(python))
-    run_case(
-        "Plan_MarksTheDriftCheckAsAnEmptyOutputCheck",
-        lambda: case_plan_marks_the_drift_check_as_an_empty_output_check(python),
-    )
-    run_case("Plan_ResolvesEveryProgram", lambda: case_plan_resolves_every_program(python))
-    run_case(
-        "Only_SelectsTheRequestedStepsWithDependencies",
-        lambda: case_only_selects_the_requested_steps_with_dependencies(python),
-    )
-    run_case("Only_FailsOnUnknownStepNames", lambda: case_only_fails_on_unknown_step_names(python))
-    run_case("Plan_ExcludesTheUnitySuites", lambda: case_plan_excludes_the_unity_suites(python))
-    run_case(
-        "LaunchFailure_SurfacesTheReasonInConsoleAndReport",
-        lambda: case_launch_failure_surfaces_the_reason_in_console_and_report(python),
-    )
-    run_case(
-        "ReportWriteFailure_IsVisibleWithoutCrashingTheGate",
-        lambda: case_report_write_failure_is_visible_without_crashing_the_gate(python),
-    )
-    run_case(
-        "LaneCrash_SurfacesTheReasonInConsoleAndReport",
-        lambda: case_lane_crash_surfaces_the_reason_in_console_and_report(python),
-    )
-    run_case(
-        "ScopedLaneTable_RewritesTheThreeLintSteps",
-        lambda: case_scoped_lane_table_rewrites_the_three_lint_steps(python),
-    )
-    run_case(
-        "ScopedLaneTable_DropsTheCSharpLaneWhenNoCSharpChanged",
-        lambda: case_scoped_lane_table_drops_the_csharp_lane_when_no_csharp_changed(python),
-    )
-    run_case(
-        "ScopedLaneTable_DropsTheUnaffectedStepsAndRemapsDependencies",
-        lambda: case_scoped_lane_table_drops_the_unaffected_steps_and_remaps_dependencies(python),
-    )
-    run_case(
-        "ScopedLaneTable_ScopesTheSuiteSelection",
-        lambda: case_scoped_lane_table_scopes_the_suite_selection(python),
-    )
-    run_case(
-        "ScopedLaneTable_KeepsWholeTreeWithoutABase",
-        lambda: case_scoped_lane_table_keeps_whole_tree_without_a_base(python),
-    )
-    run_case(
-        "PackagePathsChanged_MatchesOnlyThePayload",
-        lambda: case_package_paths_changed_matches_only_the_payload(python),
-    )
-    run_case(
-        "Execution_ScopesTheLintStepsToTheScannedDiff",
-        lambda: case_execution_scopes_the_lint_steps_to_the_scanned_diff(python),
-    )
-    run_case(
-        "Execution_SkipsTheLintStepsWhenNoCSharpChanged",
-        lambda: case_execution_skips_the_lint_steps_when_no_csharp_changed(python),
-    )
-    run_case(
-        "Execution_SkipsEveryUnaffectedStepWhenTheDiffCannotAffectThem",
-        lambda: case_execution_skips_every_unaffected_step_when_the_diff_cannot_affect_them(python),
-    )
-    run_case(
-        "Execution_ScopesTheSuiteToTheSelectedFiles",
-        lambda: case_execution_scopes_the_suite_to_the_selected_files(python),
-    )
+    run_cases([
+        ("Plan_PrintsTheFullLaneTable", lambda: case_plan_prints_the_full_lane_table(python)),
+        ("Plan_CarriesCommandsAndDependencies", lambda: case_plan_carries_commands_and_dependencies(python)),
+        ("Plan_MarksTheDriftCheckAsAnEmptyOutputCheck", lambda: case_plan_marks_the_drift_check_as_an_empty_output_check(python)),
+        ("Plan_ResolvesEveryProgram", lambda: case_plan_resolves_every_program(python)),
+        ("Only_SelectsTheRequestedStepsWithDependencies", lambda: case_only_selects_the_requested_steps_with_dependencies(python)),
+        ("Only_FailsOnUnknownStepNames", lambda: case_only_fails_on_unknown_step_names(python)),
+        ("Plan_ExcludesTheUnitySuites", lambda: case_plan_excludes_the_unity_suites(python)),
+        ("LaunchFailure_SurfacesTheReasonInConsoleAndReport", lambda: case_launch_failure_surfaces_the_reason_in_console_and_report(python)),
+        ("ReportWriteFailure_IsVisibleWithoutCrashingTheGate", lambda: case_report_write_failure_is_visible_without_crashing_the_gate(python)),
+        ("LaneCrash_SurfacesTheReasonInConsoleAndReport", lambda: case_lane_crash_surfaces_the_reason_in_console_and_report(python)),
+        ("ScopedLaneTable_RewritesTheThreeLintSteps", lambda: case_scoped_lane_table_rewrites_the_three_lint_steps(python)),
+        ("ScopedLaneTable_DropsTheCSharpLaneWhenNoCSharpChanged", lambda: case_scoped_lane_table_drops_the_csharp_lane_when_no_csharp_changed(python)),
+        ("ScopedLaneTable_DropsTheUnaffectedStepsAndRemapsDependencies", lambda: case_scoped_lane_table_drops_the_unaffected_steps_and_remaps_dependencies(python)),
+        ("ScopedLaneTable_ScopesTheSuiteSelection", lambda: case_scoped_lane_table_scopes_the_suite_selection(python)),
+        ("ScopedLaneTable_KeepsWholeTreeWithoutABase", lambda: case_scoped_lane_table_keeps_whole_tree_without_a_base(python)),
+        ("PackagePathsChanged_MatchesOnlyThePayload", lambda: case_package_paths_changed_matches_only_the_payload(python)),
+        ("Execution_ScopesTheLintStepsToTheScannedDiff", lambda: case_execution_scopes_the_lint_steps_to_the_scanned_diff(python)),
+        ("Execution_SkipsTheLintStepsWhenNoCSharpChanged", lambda: case_execution_skips_the_lint_steps_when_no_csharp_changed(python)),
+        ("Execution_SkipsEveryUnaffectedStepWhenTheDiffCannotAffectThem", lambda: case_execution_skips_every_unaffected_step_when_the_diff_cannot_affect_them(python)),
+        ("Execution_ScopesTheSuiteToTheSelectedFiles", lambda: case_execution_scopes_the_suite_to_the_selected_files(python)),
+    ])
 
     failures = failure_count()
     if failures > 0: