From 6aa794a42766b6c3394cf31cdcbdc438f5720d15 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Fri, 2 Oct 2026 11:53:52 +0100 Subject: [PATCH] test: replace implementation assertions with behavior coverage Signed-off-by: lucarlig --- .github/workflows/ci-plugin-catalog.yaml | 29 +- .github/workflows/ci-scaffold-generator.yaml | 23 +- AGENTS.md | 75 +- DEVELOPING.md | 6 +- Makefile | 2 +- README.md | 11 +- TESTING.md | 500 +- .../tests/test_build_contract.py | 15 - tests/test_install_built_wheel.py | 8 +- tests/test_plugin_catalog.py | 5132 ++--------------- tests/test_repository_policy.py | 35 + tools/templates/plugin/README.md.j2 | 4 +- .../templates/plugin/tests/test_plugin.py.j2 | 150 +- 13 files changed, 621 insertions(+), 5369 deletions(-) delete mode 100644 plugins/python/ica_metering_exporter/tests/test_build_contract.py create mode 100644 tests/test_repository_policy.py diff --git a/.github/workflows/ci-plugin-catalog.yaml b/.github/workflows/ci-plugin-catalog.yaml index a734bd26..5b8c03f2 100644 --- a/.github/workflows/ci-plugin-catalog.yaml +++ b/.github/workflows/ci-plugin-catalog.yaml @@ -5,6 +5,10 @@ on: branches: [main] paths: - "tests/test_plugin_catalog.py" + - "tests/test_repository_policy.py" + - "deny.toml" + - ".github/workflows/**" + - "Cargo.toml" - "tools/plugin_catalog.py" - "tools/validate_ci_selection.py" - "pyproject.toml" @@ -13,16 +17,15 @@ on: - "plugins/rust/python-package/*/uv.lock" - "plugins/python/**" - "plugins/rust/python-package/*/Cargo.toml" - - "plugins/rust/python-package/*/plugin-manifest.yaml" - - ".github/workflows/ci-plugin-catalog.yaml" - - ".github/workflows/ci-rust-python-package.yaml" - - ".github/workflows/release-rust-python-package.yaml" - - ".github/workflows/ci-python-package.yaml" - - ".github/workflows/release-python-package.yaml" + - "plugins/rust/python-package/*/cpex_*/plugin-manifest.yaml" pull_request: branches: [main] paths: - "tests/test_plugin_catalog.py" + - "tests/test_repository_policy.py" + - "deny.toml" + - ".github/workflows/**" + - "Cargo.toml" - "tools/plugin_catalog.py" - "tools/validate_ci_selection.py" - "pyproject.toml" @@ -31,12 +34,7 @@ on: - "plugins/rust/python-package/*/uv.lock" - "plugins/python/**" - "plugins/rust/python-package/*/Cargo.toml" - - "plugins/rust/python-package/*/plugin-manifest.yaml" - - ".github/workflows/ci-plugin-catalog.yaml" - - ".github/workflows/ci-rust-python-package.yaml" - - ".github/workflows/release-rust-python-package.yaml" - - ".github/workflows/ci-python-package.yaml" - - ".github/workflows/release-python-package.yaml" + - "plugins/rust/python-package/*/cpex_*/plugin-manifest.yaml" concurrency: group: ci-plugin-catalog-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} @@ -55,5 +53,8 @@ jobs: with: python-version: "3.12" - - name: Run plugin catalog tests - run: python3 -m unittest tests/test_plugin_catalog.py + - name: Validate repository catalog + run: python3 tools/plugin_catalog.py validate . + + - name: Run catalog behavior and repository policy tests + run: python3 -m unittest tests/test_plugin_catalog.py tests/test_repository_policy.py diff --git a/.github/workflows/ci-scaffold-generator.yaml b/.github/workflows/ci-scaffold-generator.yaml index 538ad1b8..1faca364 100644 --- a/.github/workflows/ci-scaffold-generator.yaml +++ b/.github/workflows/ci-scaffold-generator.yaml @@ -36,13 +36,12 @@ jobs: - name: Install scaffold dependencies run: python3 -m pip install jinja2 - - name: Test --help - run: python3 tools/scaffold_plugin.py --help - - name: Reject invalid hook names run: | - python3 tools/scaffold_plugin.py --non-interactive --name reject_test \ - --hooks typo_hook && exit 1 || true + if python3 tools/scaffold_plugin.py --non-interactive --name reject_test --hooks typo_hook; then + echo "Scaffolder accepted an invalid hook" >&2 + exit 1 + fi - name: Install uv and maturin run: python3 -m pip install uv==0.9.30 maturin==1.12.6 @@ -76,13 +75,6 @@ jobs: --description "CI scaffold default test plugin" \ --author "CI" - - name: Verify default plugin standalone CPEX imports - run: | - ! grep -R "mcpgateway\.plugins\.framework\|from mcpgateway\|import mcpgateway\|mcpgateway\.plugins\|gateway\.plugins" plugins/rust/python-package/ci_scaffold_default - grep -R "cpex.framework" plugins/rust/python-package/ci_scaffold_default - grep -F '"cpex>=0.1.0rc1,<0.2"' plugins/rust/python-package/ci_scaffold_default/pyproject.toml - python3 -m py_compile plugins/rust/python-package/ci_scaffold_default/cpex_ci_scaffold_default/ci_scaffold_default.py - - name: Sync default plugin working-directory: plugins/rust/python-package/ci_scaffold_default run: make sync @@ -99,13 +91,6 @@ jobs: --author "CI" \ --hooks prompt_pre_fetch,prompt_post_fetch,tool_pre_invoke,tool_post_invoke,resource_pre_fetch,resource_post_fetch,agent_pre_invoke,agent_post_invoke,http_pre_request,http_post_request,http_auth_resolve_user,http_auth_check_permission - - name: Verify all-hooks plugin standalone CPEX imports - run: | - ! grep -R "mcpgateway\.plugins\.framework\|from mcpgateway\|import mcpgateway\|mcpgateway\.plugins\|gateway\.plugins" plugins/rust/python-package/ci_scaffold_all_hooks - grep -R "cpex.framework" plugins/rust/python-package/ci_scaffold_all_hooks - grep -F '"cpex>=0.1.0rc1,<0.2"' plugins/rust/python-package/ci_scaffold_all_hooks/pyproject.toml - python3 -m py_compile plugins/rust/python-package/ci_scaffold_all_hooks/cpex_ci_scaffold_all_hooks/ci_scaffold_all_hooks.py - - name: Sync all-hooks plugin working-directory: plugins/rust/python-package/ci_scaffold_all_hooks run: make sync diff --git a/AGENTS.md b/AGENTS.md index 65aae40e..d9ace686 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,44 +16,37 @@ This is a monorepo of standalone plugin packages for the ContextForge Plugin Ext ## Testing Strategy -### Test Location by Type - -- **Unit tests**: Located within each plugin's own directory - - Python: `plugins/rust/python-package//tests/` (current hybrid) or `plugins/python//tests/` (pure Python) - - Rust: inline `mod tests` within source files (e.g., `src/lib.rs`) - - Test individual plugin functionality in isolation - - Fast, deterministic tests - - Run during plugin development and CI - - Scope: Plugin logic, Rust functions, Python bindings - -- **Plugin-framework integration tests**: Located in `plugins/rust/python-package//tests/` for Rust plugins and `plugins/tests//` for pure-Python plugins - - Test plugin integration with the local plugin framework, including the PyO3 interface for Rust plugins - - Run via `make test-integration` within the plugin directory - - Scope: Plugin entry points (including PyO3 for Rust), loading by the Python framework, hook dispatch - -- **Gateway integration tests**: Located in `mcp-context-forge/tests/integration/` - - Test plugin integration with the full gateway - - Test cross-plugin interactions - - Test plugin lifecycle management - - Scope: Plugin loading in gateway context, hook execution, framework interaction - -- **E2E tests**: Located in `mcp-context-forge/tests/e2e/` - - Test complete workflows with plugins enabled - - Test plugin behavior in realistic scenarios - - Test multi-gateway plugin coordination - - Scope: Full request/response cycles, real-world usage patterns - -### Cross-Repository Testing Coordination - -When developing a plugin: - -1. Write unit tests in the plugin's own directory (Rust: inline `mod tests` plus binding tests; pure Python: `plugins/python//tests/`) and plugin-framework integration tests in the Rust plugin's `tests/` directory or `plugins/tests//` for pure Python -2. Run local tests: `make test-all` and `make test-integration` from plugin directory -3. After plugin PR is merged, coordinate with `mcp-context-forge` team -4. Write gateway integration/E2E tests in `mcp-context-forge/tests/` -5. Ensure both repositories' CI passes before release - -See `mcp-context-forge/tests/AGENTS.md` for integration/E2E test conventions. +Keep a test when it protects an observable plugin decision, a language/framework +boundary, a release contract, a security policy, or a concrete regression. +Before adding one, identify the failure it should catch and check whether an +existing test already catches it. Test behavior at the lowest layer that owns it. + +- Rust algorithms and policy decisions: inline Rust tests in the plugin crate. +- Pure-Python logic: `plugins/python//tests/`. +- Python/Rust binding and hook behavior: `plugins/tests//`. Existing + plugins use this shared harness; it supplies controlled hook models and is + not a full gateway. The scaffolder emits plugin-local Python hook smoke tests. +- Catalog and wheel tooling behavior: `tests/`, using temporary fixture repos. +- Repository security policies: `tests/test_repository_policy.py`. +- Built-package installation: release workflows, outside the source tree. +- Full gateway and cross-plugin workflows: `mcp-context-forge/tests/integration/` + and `mcp-context-forge/tests/e2e/`. + +Do not add tests for exact workflow/Makefile text, documentation wording, +private module layout, generated stub formatting, standard-library behavior, +or generic Pydantic serialization. Build, lint, type-check, and artifact jobs +own those checks. Never commit tests whose only outcome is `pass`, `hasattr`, +or a constructor returning a non-null object. + +Use table-driven cases for distinct input classes and failure policies. Avoid +replaying a Rust algorithm's whole input matrix through Python; Python tests +should catch conversion, payload isolation, hook results, and error mapping. +Retain Redis failure/TLS tests and detection/redaction/privacy regressions: +these exercise real operational and security behavior. + +Run `make plugins-validate` for repository tooling. For plugin changes, run +`make test-all`, `make test-integration`, and the plugin's `make ci` target as +appropriate. See [TESTING.md](TESTING.md) for commands and the keep/remove decisions. ## Plugin Development Workflows @@ -85,9 +78,9 @@ The plugin framework is currently implemented in Python (`mcpgateway/plugins/fra ```bash cd plugins/rust/python-package/ # Add Rust unit tests inline in src/ using mod tests - # Add Python unit tests in tests/ - # Add plugin-framework integration tests in tests/ (run via make test-integration) - make test-all # Run Rust + Python unit tests + # Add Python boundary tests in plugins/tests// + # Extend existing tests for changed behavior + make test-all # Run Rust tests and Python hook tests make test-integration # Run plugin-framework integration tests ``` diff --git a/DEVELOPING.md b/DEVELOPING.md index 41acb794..13727f6d 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -65,7 +65,9 @@ make plugin-test PLUGIN=pii_filter `make plugins-validate` runs the same convention checks that the repo contract CI workflow runs. It runs the catalog validator plus the shared repo contract test modules: -`tests/test_plugin_catalog.py` and `tests/test_install_built_wheel.py`. +`tests/test_plugin_catalog.py`, `tests/test_install_built_wheel.py`, and +`tests/test_repository_policy.py`. See [TESTING.md](TESTING.md) for test ownership +and criteria for keeping a test. ## Secret Detection @@ -102,7 +104,7 @@ This interactive tool will: - Prompt for plugin name, description, author, and version - Let you select from 12 available hooks across 5 categories - Generate complete plugin structure with all required files -- Create comprehensive unit tests (Python and Rust) +- Add behavior tests for plugin decisions and Python/Rust boundary behavior - Set up build configuration and documentation For non-interactive mode: diff --git a/Makefile b/Makefile index be3b0d80..faf1f57d 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ plugins-list: plugins-validate: @python3 tools/plugin_catalog.py validate . - @python3 -m unittest tests/test_plugin_catalog.py tests/test_install_built_wheel.py + @python3 -m unittest discover -s tests detect-secrets-scan: ## Regenerate secrets baseline @uv tool run $(DETECT_SECRETS_SPEC) scan \ diff --git a/README.md b/README.md index c5e265aa..dcb6b314 100644 --- a/README.md +++ b/README.md @@ -46,14 +46,13 @@ Rust crates are owned by the top-level workspace in `Cargo.toml`; all Python dis Testing spans two repositories: -- **Unit tests**: within each plugin's own directory — pure Python in `plugins/python//tests/`, Rust inline via `mod tests`, and Python binding tests for Rust packages under their plugin directory -- **Plugin-framework integration tests**: `plugins/tests//` for pure-Python plugins and the Rust plugin's own `tests/` directory — test framework discovery, loading, and hook dispatch (`make test-integration`) +- **Core behavior tests**: pure Python in `plugins/python//tests/`; Rust inline via `mod tests` +- **Binding and hook tests**: `plugins/tests//` for existing plugins; generated plugins start with local Python hook smoke tests +- **Repository tooling tests**: `tests/` — catalog behavior, wheel installation, and security policies - **Gateway integration tests**: `mcp-context-forge/tests/integration/` — test plugin integration with the full gateway - **E2E tests**: `mcp-context-forge/tests/e2e/` — test complete workflows with plugins -Unit tests live in each plugin's own directory. Plugin-framework integration tests live under `plugins/tests//` for pure-Python plugins and in the plugin-local `tests/` directory for Rust plugins. Gateway integration and E2E tests live in `mcp-context-forge`. - -See [TESTING.md](TESTING.md) for detailed testing guidelines and cross-repository coordination. +Test meaningful decisions and boundary behavior at the layer that owns them. See [TESTING.md](TESTING.md) for the keep/remove criteria and commands. ## Plugin Development @@ -99,7 +98,7 @@ This interactive tool will: - Python package files (`__init__.py`, `plugin.py`) - Build configuration (`Cargo.toml`, `pyproject.toml`, `Makefile`) - Documentation (`README.md`) - - Comprehensive unit tests (Python and Rust) + - Rust configuration tests and Python hook smoke tests - Benchmark scaffolding For non-interactive mode: diff --git a/TESTING.md b/TESTING.md index 313fad5b..dfd86e57 100644 --- a/TESTING.md +++ b/TESTING.md @@ -1,182 +1,103 @@ # Testing cpex-plugins -## Testing Architecture - -Testing spans two repositories. `cpex-plugins` owns unit tests and plugin-framework integration tests; `mcp-context-forge` owns gateway integration and E2E tests. - -### Unit Tests (cpex-plugins) - -**Location**: Within each plugin's own directory -- Pure Python: `plugins/python//tests/` -- Rust package binding tests: `plugins/rust/python-package//tests/` -- Rust: inline `mod tests` within source files (e.g., `src/lib.rs`) - -**Scope**: -- Individual plugin functionality in isolation -- Rust core logic and functions -- Python bindings and entry points -- Plugin configuration validation -- Fast, deterministic tests - -**Purpose**: -- Provide fast feedback during plugin development -- Validate plugin logic independently of the gateway -- Ensure plugin contracts are met -- Test edge cases and error handling - -**Run Locally**: -```bash -# Pure Python -cd plugins/python/ -make test-all # Runs unit and plugin-framework integration tests - -# Rust with Python packaging -cd plugins/rust/python-package/ -make test-all # Runs both Rust and Python unit tests -``` - -### Plugin-Framework Integration Tests (cpex-plugins) - -**Location**: `plugins/tests//` for pure-Python plugins; the plugin-local `tests/` directory for Rust plugins - -**Scope**: -- Python plugin entry points, including the PyO3 interface for Rust plugins -- Plugin loading by the Python plugin framework -- Hook dispatch through the framework layer -- Coverage of PyO3 paths for Rust plugins (run as part of Rust coverage) - -**Purpose**: -- Validate that the Rust implementation is correctly exposed through PyO3 bindings -- Ensure the plugin framework can discover, load, and invoke the plugin -- Keep PyO3 code paths covered without requiring a full gateway - -**Run Locally**: -```bash -cd plugins/python/ # Or plugins/rust/python-package/ -make test-integration # Runs plugin-framework integration tests -``` - -The shared `plugins/tests/conftest.py` discovers selected package directories from both managed roots, checking `plugins/python/` first and then `plugins/rust/python-package/`. This Python-first lookup selects the realized package location for a slug; it does not provide a Python fallback implementation for a Rust plugin. - -### Gateway Integration Tests (mcp-context-forge) - -**Location**: `mcp-context-forge/tests/integration/` - -**Scope**: -- Plugin integration with the full gateway -- Plugin loading and initialization in gateway context -- Hook execution within the gateway framework -- Cross-plugin interactions -- Plugin lifecycle management - -**Purpose**: -- Validate plugin behavior within the gateway -- Test framework-plugin contracts at the gateway level -- Ensure plugins work together correctly -- Test plugin configuration and registration - -**Run Locally**: -```bash -cd mcp-context-forge -pytest tests/integration/ -``` - -### E2E Tests (mcp-context-forge) - -**Location**: `mcp-context-forge/tests/e2e/` - -**Scope**: -- Complete request/response workflows -- Realistic usage scenarios -- Multi-gateway plugin coordination -- Performance and load testing with plugins - -**Purpose**: -- Validate end-to-end functionality -- Test real-world usage patterns -- Ensure system-level correctness -- Catch integration issues - -**Run Locally**: -```bash -cd mcp-context-forge -pytest tests/e2e/ -``` - -## Testing Layers - -`cpex-plugins` has three local testing layers. Gateway integration and E2E tests live in `mcp-context-forge`. - -### 1. Repo Contract Tests - -These validate monorepo conventions and are enforced in CI before plugin builds run. +A test earns its place by catching a meaningful failure: an incorrect plugin +decision, a broken boundary, an unsafe release, or a known regression. Start +with the behavior and its owner. More test cases and higher coverage are not +independent goals. + +## Ownership + +| Layer | Location | What it protects | +| --- | --- | --- | +| Rust core | Inline tests in each plugin crate | Detection, redaction, limits, retries, policy decisions, and failure handling | +| Pure-Python core | `plugins/python//tests/` | Plugin decisions, request attribution, transport behavior, and privacy | +| Python/Rust and hook boundary | `plugins/tests//` | Config conversion, binding calls, payload isolation, hook results, and error mapping | +| Repository tooling | `tests/` | Catalog discovery, CI selection, release validation, coverage aggregation, and wheel selection/installation | +| Repository security policy | `tests/test_repository_policy.py` | Action SHA pinning and expiry of Cargo advisory exceptions | +| Distribution | Release workflow artifact jobs | Installation and execution of built wheels and sdists outside the source tree | +| Full gateway | `mcp-context-forge/tests/integration/` and `tests/e2e/` | Gateway lifecycle, cross-plugin interactions, and complete request flows | + +Existing plugin hook suites use `plugins/tests/conftest.py` and its controlled +hook-model shims. Real-CPEX import smoke tests check that the packages also +import with the installed framework. Passing the shim suites does not prove +full gateway compatibility. Generated plugins have local Python hook smoke +tests that run the wrapper and compiled Rust hooks against CPEX. + +## What stays, what goes + +| Keep | Remove or consolidate | +| --- | --- | +| Security detection, false-positive, redaction, and privacy regressions | Assertions about private Rust module declarations or source layout | +| Rate-limit concurrency, Redis errors, fail-open/fail-closed behavior, and TLS | Generic constructor and field-presence tests with no behavioral assertion | +| Hook dispatch, conversion, payload isolation, and observability contracts | Repeating a core algorithm's entire matrix through every wrapper layer | +| Catalog packaging invariants, mixed-language routing, canonical release tags, and coverage report validation | Hardcoded lists/counts of the current real plugins and every spelling of the same invalid input | +| Wheel platform compatibility and actual install-command behavior | Exact argparse help wording | +| Action pinning and dated advisory exception policy | Exact CI job names, step ordering, shell recipes, tool versions, or YAML whitespace | +| Building, linting, type checking, and installing actual artifacts | Reading Makefiles, stubs, templates, or documentation to assert expected text | +| One generated hook smoke case per selected hook | Empty TODO tests and tests of Pydantic's ordinary setters/serialization | + +For each new test, describe the defect that would make it fail. Extend an +existing case when it already owns that behavior. Use representative success, +failure, and boundary cases; add more inputs when they exercise a different +policy or a concrete regression. Assertions should survive an internal +refactor that preserves behavior. + +The catalog suite uses one small mixed-language fixture, direct public function +calls for policy, and a few real Git/CLI tests for I/O. It does not parse the +repository's workflow scripts or plugin source. Actual repository layout is +checked by `plugin_catalog.py validate .` in local validation and catalog CI. + +## Commands + +Repository tooling uses the standard library and needs no plugin builds: ```bash -python3 -m unittest tests/test_plugin_catalog.py tests/test_install_built_wheel.py +make plugins-validate +# Equivalent: python3 tools/plugin_catalog.py validate . +python3 -m unittest discover -s tests ``` -They verify: - -- managed plugin location under `plugins/rust/python-package/` or `plugins/python/` -- plugin manifests do not exist outside the managed root -- required files and package/module naming -- root uv-workspace membership for every plugin and top-level Cargo-workspace membership for Rust crates -- version consistency between the language-specific source (`Cargo.toml` or `pyproject.toml`) and `plugin-manifest.yaml` -- manifest `kind` consistency (`module.object`) with `[project.entry-points."cpex.plugins"]` targets (`module:object`) -- repository metadata consistency -- changed-plugin detection for CI -- canonical release tag resolution -- automatic release tag and publish workflow wiring - -### 2. Plugin Unit Tests - -Rust plugins have Rust and Python binding unit tests. Pure-Python plugins have a Python unit test suite. +For a plugin, use its existing targets: ```bash -cd plugins/rust/python-package/rate_limiter -uv sync --dev -cargo install cargo-nextest --version 0.9.133 --locked +cd plugins/rust/python-package/rate_limiter # Or plugins/python/ +make sync make install make test-all +make test-integration +make ci ``` -Set `NEXTEST_PROFILE=ci` to use the repository CI profile locally. The CI profile is defined in `.config/nextest.toml`; it disables fail-fast so all Rust test failures are reported in one run. - -Equivalent repo-level helper: +The repo helper runs the selected plugin's CI target: ```bash make plugin-test PLUGIN=rate_limiter ``` -`make plugin-test` runs the selected plugin's `make ci` target, including stub verification, build, bench compilation where configured, install, and Python tests. - -### 3. Plugin-Framework Integration Tests - -Each plugin also has integration tests between the plugin and the Python plugin framework. Pure-Python integration tests live under `plugins/tests//`; Rust integration tests live in the plugin's `tests/` directory alongside binding tests. They ensure the framework can discover, load, and invoke each plugin, and cover the PyO3 interface for Rust implementations. - -```bash -cd plugins/rust/python-package/rate_limiter -make test-integration -``` - -These tests are distinct from gateway integration tests in `mcp-context-forge`: they exercise the plugin ↔ framework boundary within this repository, without requiring a running gateway. +Rust tests use nextest. The `ci` profile in `.config/nextest.toml` disables +fail-fast. Benchmarks are compiled with `cargo nextest run --benches +-E 'kind(bench)' --no-run`; shared CI runners do not provide stable performance +measurements. Gateway suites run from the `mcp-context-forge` repository. -## 4. Rust Coverage +## Coverage and mutation checks -CI enforces at least 90% line coverage for each Rust plugin selected by the plugin catalog. The coverage job instruments Rust, runs Rust unit tests, then runs each plugin's repo-level Python integration tests so PyO3 paths are counted. +The existing Rust CI coverage gate remains 90% per selected plugin. It runs +Rust tests and the Python hook suites against instrumented PyO3 extensions. +Use coverage to find missing behavior, then add assertions that catch a real +failure. A test that only executes a line adds no useful protection. -To run the same coverage check locally for all managed Rust plugins: +To reproduce the coverage job, install `llvm-tools-preview`, +`cargo-llvm-cov` 0.8.4, and `cargo-nextest` 0.9.133, then run this in Bash from +the repository root: ```bash -rustup component add llvm-tools-preview -cargo install cargo-llvm-cov --version 0.8.4 --locked -cargo install cargo-nextest --version 0.9.133 --locked mkdir -p coverage CARGO_PACKAGES="$(python3 tools/plugin_catalog.py ci-selection-field . all '' '' cargo_packages)" RUST_PLUGINS="$(python3 tools/plugin_catalog.py ci-selection-field . all '' '' rust_plugins)" -mapfile -t cargo_packages < <(python3 -c 'import json, os; [print(package) for package in json.loads(os.environ["CARGO_PACKAGES"])]') -mapfile -t rust_plugins < <(python3 -c 'import json, os; [print(plugin) for plugin in json.loads(os.environ["RUST_PLUGINS"])]') +export CARGO_PACKAGES RUST_PLUGINS +mapfile -t cargo_packages < <(python3 -c 'import json, os; [print(p) for p in json.loads(os.environ["CARGO_PACKAGES"])]') +mapfile -t rust_plugins < <(python3 -c 'import json, os; [print(p) for p in json.loads(os.environ["RUST_PLUGINS"])]') cargo_args=() for package in "${cargo_packages[@]}"; do cargo_args+=("-p" "${package}") @@ -190,278 +111,29 @@ export LLVM_PROFILE_FILE="${CARGO_TARGET_DIR}/cpex-plugins-%p-%10m.profraw" mkdir -p "${CARGO_TARGET_DIR}" for plugin in "${rust_plugins[@]}"; do (cd "plugins/rust/python-package/${plugin}" && make sync && uv run maturin develop) -done -for plugin in "${rust_plugins[@]}"; do (cd "plugins/rust/python-package/${plugin}" && make test-integration) done env -u CARGO_TARGET_DIR -u CARGO_LLVM_COV_BUILD_DIR -u CARGO_LLVM_COV_TARGET_DIR -u LLVM_PROFILE_FILE cargo llvm-cov report "${cargo_args[@]}" --cobertura --output-path coverage/cobertura.xml python3 tools/plugin_catalog.py coverage-check . coverage/cobertura.xml 90.00 "${RUST_PLUGINS}" ``` -Rust unit tests use `cargo nextest run`. Coverage uses `cargo llvm-cov nextest --no-report` for the Rust test phase, then runs pytest before generating the final report so PyO3 paths stay covered. CI uses the `ci` nextest profile, which disables fail-fast and prints failure output immediately and again at the end. Nextest does not run Rust doctests; this repo currently has no Rust doctest code blocks, so there is no separate doctest step. - -Criterion benchmarks are verified in CI with `cargo nextest run --benches -E 'kind(bench)' --no-run`, which compiles benchmark test targets without rerunning normal unit tests or collecting noisy performance measurements on shared CI runners. - -## 5. Mutation Testing - -Mutation testing runs in PR CI on Ubuntu for Rust code touched by the pull request diff. It is also available locally through cargo-mutants and runs Rust tests with nextest. +Mutation CI checks changed Rust source, including relevant dependents of the +framework bridge. It uses `cargo-mutants` 27.0.0 with nextest and the `mutants` +profile. Tooling-only changes do not select mutation jobs. Local commands: ```bash -cargo install cargo-nextest --version 0.9.133 --locked -cargo install cargo-mutants --version 27.0.0 --locked make plugin-mutants-list PLUGIN=retry_with_backoff make plugin-mutants PLUGIN=retry_with_backoff ``` -`.cargo/mutants.toml` sets `test_tool = "nextest"`, selects the `mutants` nextest profile, and keeps `cap_lints = false` so Rust warnings are not downgraded during mutant builds. The `mutants` profile keeps fail-fast enabled because cargo-mutants only needs one failing test to mark a mutant as caught. CI installs `cargo-mutants` with `cargo install cargo-mutants --version 27.0.0 --locked` and runs `cargo mutants "${cargo_args[@]}"`, using `--in-diff cargo-mutants.diff` for Rust source changes and full-package mutation for mutation-tooling config changes. - -## Cross-Repository Testing Workflow - -### Development Workflow - -1. **Develop Plugin in cpex-plugins**: - ```bash - cd cpex-plugins/plugins/python/ # Or plugins/rust/python-package/ - # Implement plugin logic - # Write unit tests in the plugin-local tests/ - # Write pure-Python integration tests in cpex-plugins/plugins/tests// - make test-all # Run unit tests - make test-integration # Run plugin-framework integration tests - ``` - -2. **Create PR in cpex-plugins**: - - Include unit tests and plugin-framework integration tests - - Ensure `make ci` passes - - Get PR reviewed and merged - -3. **Coordinate with mcp-context-forge**: - - Notify mcp-context-forge team of new plugin - - Discuss integration test requirements - - Plan E2E test scenarios - -4. **Write Integration Tests in mcp-context-forge**: - ```bash - cd mcp-context-forge - # Install plugin: pip install cpex- - # Configure in plugins/config.yaml - # Write tests in tests/integration/ - pytest tests/integration/ - ``` - -5. **Write E2E Tests in mcp-context-forge**: - ```bash - cd mcp-context-forge - # Write tests in tests/e2e/ - pytest tests/e2e/ - ``` - -6. **Create PR in mcp-context-forge**: - - Include integration and E2E tests - - Ensure all tests pass - - Get PR reviewed and merged - -7. **Release**: - - Tag plugin in cpex-plugins: `-v` - - Update mcp-context-forge dependencies - - Deploy with new plugin version - -### Testing Coordination Guidelines - -**When to Write Unit Tests (cpex-plugins)**: -- Testing plugin logic in isolation -- Testing Rust functions and algorithms -- Testing Python bindings -- Testing configuration validation -- Testing error handling and edge cases - -**When to Write Plugin-Framework Integration Tests (cpex-plugins)**: -- Testing PyO3 entry points end-to-end -- Testing plugin loading by the Python framework -- Testing hook dispatch through the framework layer -- Ensuring PyO3 paths are covered in Rust coverage - -**When to Write Gateway Integration Tests (mcp-context-forge)**: -- Testing plugin loading and initialization in the gateway -- Testing hook execution in the full gateway framework -- Testing plugin interactions with gateway services -- Testing cross-plugin behavior -- Testing plugin lifecycle (enable/disable/reload) - -**When to Write E2E Tests (mcp-context-forge)**: -- Testing complete request/response flows -- Testing realistic usage scenarios -- Testing performance with plugins enabled -- Testing multi-gateway coordination -- Testing production-like configurations - -### CI Coordination - -**cpex-plugins CI**: -- Runs repo contract tests -- Runs plugin unit tests -- Runs plugin-framework integration tests (`make test-integration`) -- Builds and packages plugins -- Uses separate `ci-python-package.yaml` and `ci-rust-python-package.yaml` workflows fed by language-specific catalog selections -- On `main`, creates release tags for plugin version bumps only after required checks are green -- Invokes the release workflow for PyPI publishing after tag creation - -**mcp-context-forge CI**: -- Runs integration tests with latest plugin versions -- Runs E2E tests with plugins enabled -- Validates plugin compatibility -- Tests plugin upgrades - -### Test Coverage Expectations - -**Unit Tests (cpex-plugins)**: -- Aim for >90% code coverage of plugin logic -- Cover all public APIs and entry points -- Test error paths and edge cases -- Fast execution (<1 second per test) - -**Integration Tests (mcp-context-forge)**: -- Cover all plugin hooks -- Test plugin configuration variations -- Test plugin interactions -- Moderate execution time (<5 seconds per test) - -**E2E Tests (mcp-context-forge)**: -- Cover critical user workflows -- Test realistic scenarios -- Test performance characteristics -- Slower execution acceptable (seconds to minutes) - -## CI Behavior - -Repo contract tests run in their own CI workflow. Separate pure-Python and Rust plugin CI workflows use the same dual-root catalog to select affected jobs; Rust-only integration and coverage behavior remains in the Rust workflow. - -Per-plugin build/test jobs are then scoped by the plugin catalog: - -- plugin-only changes run only the affected plugin jobs -- shared workflow, workspace, root orchestration, docs, test, and tool changes run all managed plugin jobs - -For pull requests with plugin version bumps, each language-specific CI workflow -invokes its matching release workflow to build and test the target package with -publishing disabled. On `main`, each language-specific CI workflow creates tags -only after its required checks pass, then invokes its matching release workflow -with PyPI publishing enabled. Rust CI retains additional security, mutation, -coverage, and documentation gates. The matching release workflow validates the -tag and plugin metadata before any artifact is published. - -## Testing Best Practices - -### Unit Tests - -- **Fast**: Each test should complete in milliseconds -- **Isolated**: No external dependencies (network, filesystem, database) -- **Deterministic**: Same input always produces same output -- **Focused**: Test one thing per test -- **Clear**: Test names describe what is being tested - -### Integration Tests - -- **Realistic**: Use actual gateway framework components -- **Scoped**: Test specific integration points -- **Stable**: Use test fixtures and mocks for external services -- **Documented**: Explain what integration is being tested - -### E2E Tests - -- **Complete**: Test full workflows from start to finish -- **Representative**: Use realistic data and scenarios -- **Robust**: Handle timing and async operations correctly -- **Maintainable**: Use page objects and test helpers - -## Running Tests - -### Local Development - -```bash -# In cpex-plugins -cd plugins/python/ # Or plugins/rust/python-package/ -make test-all # Run language-appropriate unit tests -make test-integration # Run plugin-framework integration tests - -# In mcp-context-forge -cd mcp-context-forge -pytest tests/integration/ # Run gateway integration tests -pytest tests/e2e/ # Run E2E tests -``` - -### CI Pipeline - -```bash -# cpex-plugins CI -make plugins-validate # Validate repo structure -make plugin-test PLUGIN= # Run unit tests for specific plugin -# make test-integration is run as part of the coverage job - -# mcp-context-forge CI -make test # Run unit tests -pytest tests/integration/ # Run gateway integration tests -pytest tests/e2e/ # Run E2E tests -``` - -## Debugging Test Failures - -### Unit Test Failures (cpex-plugins) - -1. Run tests locally: `make test-all` -2. Check Rust test output: `cargo test -- --nocapture` -3. Check Python test output: `pytest -v` -4. Use debugger: `rust-gdb` or `pdb` - -### Plugin-Framework Integration Test Failures (cpex-plugins) - -1. Run tests locally: `make test-integration` -2. Check PyO3 binding output: `pytest -v tests/` -3. Verify Rust extension is built: `make install` -4. Check framework loading: `pytest -vv tests/` - -### Gateway Integration Test Failures (mcp-context-forge) - -1. Check plugin installation: `pip list | grep cpex` -2. Verify plugin configuration: `cat plugins/config.yaml` -3. Check gateway logs: `tail -f logs/gateway.log` -4. Run with verbose output: `pytest -vv tests/integration/` - -### E2E Test Failures (mcp-context-forge) - -1. Check full system logs -2. Verify all services are running -3. Check network connectivity -4. Run with debug logging: `LOG_LEVEL=DEBUG pytest tests/e2e/` - -## Test Documentation - -For detailed testing conventions in mcp-context-forge, see: -- `mcp-context-forge/tests/AGENTS.md` - Testing conventions and workflows -- `mcp-context-forge/plugins/AGENTS.md` - Plugin framework testing - -## Future: Pure Rust Testing - -After the plugin framework is migrated to Rust: - -### Unit Tests (cpex-plugins) - -```bash -cd plugins/rust/ -cargo test # Run Rust tests -cargo test -- --nocapture # With output -``` - -### Integration Tests (mcp-context-forge) - -```bash -cd mcp-context-forge -cargo test --test integration # Run integration tests -``` +## CI and releases -### E2E Tests (mcp-context-forge) - -```bash -cd mcp-context-forge -cargo test --test e2e # Run E2E tests -``` +The catalog selects affected plugins and splits Rust and Python jobs. Plugin +changes stay scoped; shared workspace, tooling, and harness changes select all +plugins. Repository test-only changes run the tooling suite. -Python test infrastructure will be removed after framework migration. +Plugin CI retains formatting, linting, type checks, Rust tests, hook suites, +security checks, coverage, and artifact builds. Version bumps on PRs invoke the +matching release workflow with publishing disabled. On `main`, release tags +are created after required checks pass. Release workflows validate catalog +metadata and exercise built distributions before publishing. diff --git a/plugins/python/ica_metering_exporter/tests/test_build_contract.py b/plugins/python/ica_metering_exporter/tests/test_build_contract.py deleted file mode 100644 index 666e547f..00000000 --- a/plugins/python/ica_metering_exporter/tests/test_build_contract.py +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Build contract tests for plugin-local distribution artifacts.""" - -from pathlib import Path - - -def test_build_target_writes_plugin_local_artifacts() -> None: - # Given the plugin Makefile executed by the CI target. - makefile_path = Path(__file__).parents[1] / "Makefile" - - # When its build recipe is inspected. - makefile = makefile_path.read_text() - - # Then uv targets the plugin-local distribution directory. - assert "uv build --project . --out-dir ./dist" in makefile diff --git a/tests/test_install_built_wheel.py b/tests/test_install_built_wheel.py index 96ca7529..80b9789a 100644 --- a/tests/test_install_built_wheel.py +++ b/tests/test_install_built_wheel.py @@ -1,3 +1,6 @@ +# Copyright 2026 +# SPDX-License-Identifier: Apache-2.0 + import importlib.util import os import subprocess @@ -27,11 +30,6 @@ def run_install_built_wheel(*args: str) -> subprocess.CompletedProcess[str]: class InstallBuiltWheelTests(unittest.TestCase): - def test_help_describes_compatibility_aware_selection(self) -> None: - result = run_install_built_wheel("--help") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("best matching compatible wheel", result.stdout) - def test_prints_only_matching_wheel_when_multiple_exist(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: wheel_dir = Path(tmpdir) diff --git a/tests/test_plugin_catalog.py b/tests/test_plugin_catalog.py index d0e02b1a..186a6ccc 100644 --- a/tests/test_plugin_catalog.py +++ b/tests/test_plugin_catalog.py @@ -1,4772 +1,454 @@ -import json -import os -import ast -import shutil -import subprocess -import sys -import tempfile -import textwrap -import tomllib -import unittest -from datetime import date -from pathlib import Path -import re - -from tools.plugin_catalog import CatalogError, discover_plugins - - -REPO_ROOT = Path(__file__).resolve().parents[1] -SCRIPT = REPO_ROOT / "tools" / "plugin_catalog.py" -CI_SELECTION_VALIDATOR = REPO_ROOT / "tools" / "validate_ci_selection.py" -JsonValue = str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"] - - -def run_catalog(*args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: - return subprocess.run( - ["python3", str(SCRIPT), *args], - cwd=cwd or REPO_ROOT, - text=True, - capture_output=True, - check=False, - ) - - -class PluginCatalogTests(unittest.TestCase): - def _assert_payload_contains( - self, - payload: dict[str, JsonValue], - expected: dict[str, JsonValue], - ) -> None: - self.assertEqual({key: payload[key] for key in expected}, expected) - - def _extract_workflow_job_section(self, workflow: str, job_name: str) -> str: - lines = workflow.splitlines() - in_jobs = False - job_header = f" {job_name}:" - section_lines: list[str] = [] - - for line in lines: - if line == "jobs:": - in_jobs = True - continue - if not in_jobs: - continue - if line.startswith(" ") and not line.startswith(" "): - if section_lines: - break - if line == job_header: - section_lines.append(line) - continue - if section_lines: - section_lines.append(line) - - self.assertTrue(section_lines, f"expected to find workflow job {job_name!r}") - return "\n".join(section_lines) + "\n" - - def _extract_workflow_step_section( - self, - workflow: str, - job_name: str, - *, - step_id: str | None = None, - step_name: str | None = None, - ) -> str: - job_section = self._extract_workflow_job_section(workflow, job_name) - lines = job_section.splitlines() - step_header = None - if step_id is not None: - step_header = f" - id: {step_id}" - elif step_name is not None: - step_header = f" - name: {step_name}" - else: - raise AssertionError("step_id or step_name is required") - - section_lines: list[str] = [] - for line in lines: - if line.startswith(" - ") and section_lines: - break - if line == step_header: - section_lines.append(line) - continue - if section_lines: - section_lines.append(line) - - self.assertTrue( - section_lines, - f"expected to find workflow step {step_id or step_name!r} in {job_name!r}", - ) - return "\n".join(section_lines) + "\n" - - def _extract_workflow_step_run( - self, - workflow: str, - job_name: str, - *, - step_id: str | None = None, - step_name: str | None = None, - ) -> str: - step_section = self._extract_workflow_step_section( - workflow, job_name, step_id=step_id, step_name=step_name - ) - lines = step_section.splitlines() - run_lines: list[str] = [] - in_run = False - for line in lines: - if line == " run: |": - in_run = True - continue - if line.startswith(" run: "): - return line.removeprefix(" run: ") + "\n" - if in_run: - if line == "": - run_lines.append("") - continue - if not line.startswith(" "): - break - run_lines.append(line.removeprefix(" ")) - - self.assertTrue( - run_lines, - f"expected workflow step {step_id or step_name!r} in {job_name!r} to have a run script", - ) - return "\n".join(run_lines) + "\n" - - def _assert_workflow_needs_outputs_declared( - self, - workflow: str, - producer_job: str, - ) -> None: - job_section = self._extract_workflow_job_section(workflow, producer_job) - outputs_section = job_section.split(" outputs:\n", maxsplit=1)[1].split( - " steps:\n", maxsplit=1 - )[0] - declared = set(re.findall(r"^ ([a-z0-9_-]+):", outputs_section, re.MULTILINE)) - referenced = set( - re.findall(rf"needs\.{re.escape(producer_job)}\.outputs\.([a-z0-9_-]+)", workflow) - ) - self.assertEqual(referenced - declared, set()) - - def _source_tree_has_extension(self, package_dir: Path, module_name: str) -> bool: - return any(package_dir.glob(f"{module_name}*.so")) or any( - package_dir.glob(f"{module_name}*.pyd") - ) - - def _create_plugin(self, root: Path, slug: str) -> Path: - plugin_dir = root / "plugins" / "rust" / "python-package" / slug - package_dir = plugin_dir / f"cpex_{slug}" - class_name = f"{slug.title().replace('_', '')}Plugin" - manifest_kind = f"cpex_{slug}.{slug}.{class_name}" - entry_point_kind = f"cpex_{slug}.{slug}:{class_name}" - package_dir.mkdir(parents=True) - (plugin_dir / "pyproject.toml").write_text( - ( - f"[project]\nname = \"cpex-{slug.replace('_', '-')}\"\ndynamic = [\"version\"]\n\n" - "[project.entry-points.\"cpex.plugins\"]\n" - f"{slug} = \"{entry_point_kind}\"\n\n" - "[tool.maturin]\n" - f"module-name = \"cpex_{slug}.{slug}_rust\"\n" - "python-source = \".\"\n" - ) - ) - (plugin_dir / "Cargo.toml").write_text( - f"[package]\nname = \"{slug}\"\nversion = \"0.0.1\"\nrepository = \"https://github.com/IBM/cpex-plugins\"\n" - ) - (plugin_dir / "Makefile").write_text("all:\n\t@true\n") - (plugin_dir / "README.md").write_text(f"# {slug}\n") - (package_dir / "__init__.py").write_text("") - (package_dir / "plugin-manifest.yaml").write_text( - f'description: "{slug}"\nauthor: "ContextForge Team"\nversion: "0.0.1"\nkind: "{manifest_kind}"\navailable_hooks:\n - "tool_pre_invoke"\n' - ) - return plugin_dir - - def _create_python_plugin( - self, - root: Path, - slug: str, - *, - version: str = "0.2.0", - add_workspace_member: bool = True, - ) -> Path: - rust_root = root / "plugins" / "rust" / "python-package" - rust_root.mkdir(parents=True, exist_ok=True) - if not (root / "Cargo.toml").exists(): - (root / "Cargo.toml").write_text("[workspace]\nmembers = []\n") - - plugin_dir = root / "plugins" / "python" / slug - package_dir = plugin_dir / f"cpex_{slug}" - class_name = f"{slug.title().replace('_', '')}Plugin" - manifest_kind = f"cpex_{slug}.plugin.{class_name}" - package_dir.mkdir(parents=True) - (plugin_dir / "pyproject.toml").write_text( - f'[project]\nname = "cpex-{slug.replace("_", "-")}"\nversion = "{version}"\n\n' - '[project.entry-points."cpex.plugins"]\n' - f'{slug} = "cpex_{slug}.plugin:{class_name}"\n' - ) - (plugin_dir / "Makefile").write_text("all:\n\t@true\n") - (plugin_dir / "README.md").write_text(f"# {slug}\n") - (package_dir / "__init__.py").write_text("") - (package_dir / "plugin-manifest.yaml").write_text( - f'description: "{slug}"\nauthor: "ContextForge Team"\nversion: "{version}"\n' - f'kind: "{manifest_kind}"\navailable_hooks:\n - "tool_pre_invoke"\n' - ) - members = [f"plugins/python/{slug}"] if add_workspace_member else [] - rendered_members = ", ".join(json.dumps(member) for member in members) - (root / "pyproject.toml").write_text( - f"[tool.uv.workspace]\nmembers = [{rendered_members}]\n" - ) - return plugin_dir - - def _parse_manifest_defaults(self, manifest_path: Path) -> dict[str, object]: - defaults: dict[str, object] = {} - in_defaults = False - for line in manifest_path.read_text().splitlines(): - if line == "default_configs:": - in_defaults = True - continue - if in_defaults and not line.startswith(" "): - break - if not in_defaults: - continue - key, _, raw_value = line.strip().partition(":") - value = raw_value.strip() - if value in {"true", "false"}: - defaults[key] = value == "true" - elif value == "[]": - defaults[key] = [] - elif value.startswith('"') or value.startswith("'"): - defaults[key] = ast.literal_eval(value) - else: - defaults[key] = int(value) - return defaults - - def _extract_pii_runtime_defaults(self) -> dict[str, object]: - config_text = ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "pii_filter" - / "src" - / "config.rs" - ).read_text() - default_impl = config_text.split("impl Default for PIIConfig {", maxsplit=1)[1] - constants: dict[str, int] = {} - for name in ( - "DEFAULT_MAX_TEXT_BYTES", - "DEFAULT_MAX_NESTED_DEPTH", - "DEFAULT_MAX_COLLECTION_ITEMS", - ): - match = re.search(rf"{name}: usize = ([^;]+);", config_text) - assert match is not None - expr = match.group(1).replace("_", "") - constants[name] = eval(expr, {"__builtins__": {}}, {}) - - defaults: dict[str, object] = {} - keys = ( - "detect_ssn", - "detect_bsn", - "detect_credit_card", - "detect_email", - "detect_phone", - "detect_ip_address", - "detect_date_of_birth", - "detect_passport", - "detect_driver_license", - "detect_bank_account", - "detect_medical_record", - "default_mask_strategy", - "redaction_text", - "block_on_detection", - "log_detections", - "include_detection_details", - "max_text_bytes", - "max_nested_depth", - "max_collection_items", - "custom_patterns", - "whitelist_patterns", - ) - for key in keys: - match = re.search(rf"{key}: ([^,]+),", default_impl) - assert match is not None - raw = match.group(1).strip() - if raw in {"true", "false"}: - defaults[key] = raw == "true" - elif raw == "MaskingStrategy::Redact": - defaults[key] = "redact" - elif raw == '\"[REDACTED]\".to_string()': - defaults[key] = "[REDACTED]" - elif raw in constants: - defaults[key] = constants[raw] - elif raw == "Vec::new()": - defaults[key] = [] - else: - raise AssertionError(f"Unhandled default expression for {key}: {raw}") - return defaults - - def test_workflow_actions_are_pinned_to_commit_shas(self) -> None: - uses_pattern = re.compile(r"^\s*-?\s*uses:\s+(.+?)\s*$") - pinned_pattern = re.compile( - r"^\s*-?\s*uses:\s+[^@\s]+@[0-9a-f]{40}\s+#\s+\S+\s*$" - ) - for workflow_path in sorted((REPO_ROOT / ".github" / "workflows").glob("*.y*ml")): - for line_number, line in enumerate(workflow_path.read_text().splitlines(), start=1): - match = uses_pattern.match(line) - if match is None: - continue - if match.group(1).startswith("./"): - continue - self.assertRegex( - line, - pinned_pattern, - f"{workflow_path.relative_to(REPO_ROOT)}:{line_number} should pin third-party actions to commit SHAs", - ) - - def test_repo_validates_managed_plugins_layout(self) -> None: - result = run_catalog("validate", str(REPO_ROOT)) - self.assertEqual(result.returncode, 0, result.stderr) - - def test_cargo_deny_advisory_ignores_are_not_expired(self) -> None: - deny_config = tomllib.loads((REPO_ROOT / "deny.toml").read_text()) - ignores = deny_config.get("advisories", {}).get("ignore", []) - for entry in ignores: - if not isinstance(entry, dict): - continue - reason = str(entry.get("reason", "")) - match = re.search(r"expires (\d{4}-\d{2}-\d{2})", reason) - if match is None: - continue - expiry = date.fromisoformat(match.group(1)) - self.assertLessEqual( - date.today(), - expiry, - f"{entry.get('id', entry)} advisory ignore expired on {expiry}", - ) - - def test_repo_centralizes_shared_cargo_dependencies(self) -> None: - cargo = tomllib.loads((REPO_ROOT / "Cargo.toml").read_text()) - workspace_deps = cargo["workspace"]["dependencies"] - - self.assertIn("cpex_framework_bridge", workspace_deps) - self.assertIn("criterion", workspace_deps) - self.assertIn("log", workspace_deps) - self.assertIn("mutants", workspace_deps) - self.assertIn("pyo3", workspace_deps) - self.assertIn("pyo3-log", workspace_deps) - self.assertIn("pyo3-stub-gen", workspace_deps) - self.assertIn("regex", workspace_deps) - self.assertIn("serde", workspace_deps) - self.assertIn("serde_json", workspace_deps) - self.assertIn("thiserror", workspace_deps) - self.assertNotIn("pyo3-async-runtimes", workspace_deps) - self.assertNotIn("rand", workspace_deps) - self.assertNotIn("tokio", workspace_deps) - - expected_plugin_deps = { - "encoded_exfil_detection": { - "dependencies": { - "cpex_framework_bridge": {"workspace": True}, - "log": {"workspace": True}, - "pyo3": {"workspace": True}, - "pyo3-log": {"workspace": True}, - "pyo3-stub-gen": {"workspace": True, "optional": True}, - "regex": {"workspace": True}, - "serde_json": {"workspace": True}, - }, - "dev-dependencies": { - "criterion": {"workspace": True}, - }, - }, - "pii_filter": { - "dependencies": { - "cpex_framework_bridge": {"workspace": True}, - "log": {"workspace": True}, - "pyo3": {"workspace": True}, - "pyo3-log": {"workspace": True}, - "pyo3-stub-gen": {"workspace": True, "optional": True}, - "regex": {"workspace": True}, - "serde": {"workspace": True}, - "serde_json": {"workspace": True}, - "thiserror": {"workspace": True}, - }, - "dev-dependencies": { - "criterion": {"workspace": True}, - }, - }, - "rate_limiter": { - "dependencies": { - "cpex_framework_bridge": {"workspace": True}, - "log": {"workspace": True}, - "mutants": {"workspace": True}, - "pyo3": {"workspace": True}, - "pyo3-log": {"workspace": True}, - "pyo3-stub-gen": {"workspace": True, "optional": True}, - "thiserror": {"workspace": True}, - }, - "dev-dependencies": { - "criterion": {"workspace": True}, - }, - }, - "retry_with_backoff": { - "dependencies": { - "cpex_framework_bridge": {"workspace": True}, - "log": {"workspace": True}, - "mutants": {"workspace": True}, - "pyo3": {"workspace": True}, - "pyo3-log": {"workspace": True}, - "pyo3-stub-gen": {"workspace": True, "optional": True}, - "serde": {"workspace": True}, - "serde_json": {"workspace": True}, - }, - "dev-dependencies": {}, - }, - "secrets_detection": { - "dependencies": { - "cpex_framework_bridge": {"workspace": True}, - "log": {"workspace": True}, - "pyo3": {"workspace": True}, - "pyo3-log": {"workspace": True}, - "pyo3-stub-gen": {"workspace": True, "optional": True}, - "regex": {"workspace": True}, - "serde_json": {"workspace": True}, - }, - "dev-dependencies": { - "criterion": {"workspace": True}, - }, - }, - "url_reputation": { - "dependencies": { - "log": {"workspace": True}, - "pyo3": {"workspace": True}, - "pyo3-log": {"workspace": True}, - "regex": {"workspace": True}, - }, - "dev-dependencies": { - "criterion": {"workspace": True}, - }, - }, - } - - for slug, expected_sections in expected_plugin_deps.items(): - plugin_cargo = tomllib.loads( - ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / slug - / "Cargo.toml" - ).read_text() - ) - for section_name, expected_deps in expected_sections.items(): - actual_section = plugin_cargo.get(section_name, {}) - self.assertIsInstance(actual_section, dict) - for dependency_name, expected_value in expected_deps.items(): - self.assertEqual( - actual_section.get(dependency_name), - expected_value, - f"{slug} should source {dependency_name} from workspace {section_name}", - ) - - def test_plugin_scaffold_templates_use_standalone_cpex_imports(self) -> None: - template_root = REPO_ROOT / "tools" / "templates" / "plugin" - rendered_source = "\n".join( - path.read_text() - for path in sorted(template_root.rglob("*")) - if path.is_file() - ) - - self.assertNotIn("mcpgateway", rendered_source) - self.assertIn("cpex.framework", rendered_source) - self.assertIn('"cpex>=0.1.0rc1,<0.2"', rendered_source) - - def test_plugin_scaffold_generator_renders_standalone_cpex_imports(self) -> None: - try: - import jinja2 # noqa: F401 - except ImportError: - self.skipTest("jinja2 is required to render scaffold templates") - - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "plugins" / "rust" / "python-package").mkdir(parents=True) - (root / "tools").mkdir() - shutil.copytree(REPO_ROOT / "tools" / "templates", root / "tools" / "templates") - (root / "Cargo.toml").write_text( - textwrap.dedent( - """ - [workspace] - members = [] - resolver = "2" - - [workspace.package] - edition = "2024" - authors = ["Test"] - license = "Apache-2.0" - repository = "https://example.invalid" - - [workspace.dependencies] - cpex_framework_bridge = { path = "crates/framework_bridge" } - criterion = "0.8" - log = "0.4" - pyo3 = { version = "0.28.2", features = ["abi3-py311"] } - pyo3-log = "0.13" - pyo3-stub-gen = "0.22.1" - regex = "1" - """ - ).lstrip() - ) - result = subprocess.run( - [ - sys.executable, - str(REPO_ROOT / "tools" / "scaffold_plugin.py"), - "--non-interactive", - "--name", - "ci_scaffold_all_hooks", - "--description", - "CI scaffold all hooks", - "--author", - "CI", - "--hooks", - "prompt_pre_fetch,prompt_post_fetch,tool_pre_invoke,tool_post_invoke,resource_pre_fetch,resource_post_fetch,agent_pre_invoke,agent_post_invoke,http_pre_request,http_post_request,http_auth_resolve_user,http_auth_check_permission", - ], - cwd=root, - text=True, - capture_output=True, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - - plugin_dir = root / "plugins" / "rust" / "python-package" / "ci_scaffold_all_hooks" - generated_source = "\n".join(path.read_text() for path in plugin_dir.rglob("*") if path.is_file()) - self.assertNotIn("mcpgateway", generated_source) - self.assertIn("cpex.framework", generated_source) - self.assertIn('"cpex>=0.1.0rc1,<0.2"', generated_source) - cargo_manifest = tomllib.loads((plugin_dir / "Cargo.toml").read_text()) - self.assertEqual(cargo_manifest["features"]["default"], []) - self.assertEqual(cargo_manifest["features"]["stub-gen"], ["dep:pyo3-stub-gen"]) - self.assertEqual( - cargo_manifest["dependencies"]["pyo3-stub-gen"], - {"workspace": True, "optional": True}, - ) - self.assertEqual(cargo_manifest["bin"][0]["required-features"], ["stub-gen"]) - - compile_result = subprocess.run( - [ - sys.executable, - "-m", - "py_compile", - str(plugin_dir / "cpex_ci_scaffold_all_hooks" / "ci_scaffold_all_hooks.py"), - ], - text=True, - capture_output=True, - check=False, - ) - self.assertEqual(compile_result.returncode, 0, compile_result.stderr) - - def test_repo_lists_all_managed_plugins(self) -> None: - result = run_catalog("list", str(REPO_ROOT)) - self.assertEqual(result.returncode, 0, result.stderr) - - payload = json.loads(result.stdout) - self.assertEqual( - {entry["slug"] for entry in payload["plugins"]}, - { - "encoded_exfil_detection", - "ica_metering_exporter", - "output_length_guard", - "pii_filter", - "rate_limiter", - "retry_with_backoff", - "secrets_detection", - "sql_sanitizer", - "url_reputation", - }, - ) - by_slug = {entry["slug"]: entry for entry in payload["plugins"]} - self.assertEqual( - {slug: entry["module_name"] for slug, entry in by_slug.items()}, - { - "encoded_exfil_detection": "cpex_encoded_exfil_detection", - "ica_metering_exporter": "cpex_ica_metering_exporter", - "output_length_guard": "cpex_output_length_guard", - "pii_filter": "cpex_pii_filter", - "rate_limiter": "cpex_rate_limiter", - "retry_with_backoff": "cpex_retry_with_backoff", - "secrets_detection": "cpex_secrets_detection", - "sql_sanitizer": "cpex_sql_sanitizer", - "url_reputation": "cpex_url_reputation", - }, - ) - self.assertEqual( - {slug: entry["kind"] for slug, entry in by_slug.items()}, - { - "encoded_exfil_detection": "cpex_encoded_exfil_detection.encoded_exfil_detection.EncodedExfilDetectorPlugin", - "ica_metering_exporter": "cpex_ica_metering_exporter.plugin.IcaMeteringExporterPlugin", - "output_length_guard": "cpex_output_length_guard.output_length_guard.OutputLengthGuardPlugin", - "pii_filter": "cpex_pii_filter.pii_filter.PIIFilterPlugin", - "rate_limiter": "cpex_rate_limiter.rate_limiter.RateLimiterPlugin", - "retry_with_backoff": "cpex_retry_with_backoff.retry_with_backoff.RetryWithBackoffPlugin", - "secrets_detection": "cpex_secrets_detection.secrets_detection.SecretsDetectionPlugin", - "sql_sanitizer": "cpex_sql_sanitizer.sql_sanitizer.SQLSanitizerPlugin", - "url_reputation": "cpex_url_reputation.url_reputation.URLReputationPlugin", - }, - ) - - def test_validator_rejects_manifest_missing_kind(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - package_dir = plugin_dir / "cpex_demo_plugin" - (package_dir / "plugin-manifest.yaml").write_text( - textwrap.dedent( - """ - description: "Demo plugin" - author: "ContextForge Team" - version: "0.0.1" - available_hooks: - - "tool_pre_invoke" - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("missing kind", result.stderr.lower()) - - def test_validator_rejects_invalid_plugin_slug(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo-plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - (root / "plugins" / "rust" / "python-package" / "demo-plugin").mkdir( - parents=True - ) - - result = run_catalog("validate", str(root)) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("plugin slug must match", result.stderr) - - def test_validator_rejects_cargo_package_name_drift(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "Cargo.toml").write_text( - '[package]\nname = "wrong_name"\nversion = "0.0.1"\n' - 'repository = "https://github.com/IBM/cpex-plugins"\n' - ) - - result = run_catalog("validate", str(root)) - - self.assertNotEqual(result.returncode, 0) - self.assertIn( - "Cargo.toml [package].name must be demo_plugin", result.stderr - ) - - def test_validator_rejects_manifest_kind_with_trailing_junk(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - package_dir = plugin_dir / "cpex_demo_plugin" - (package_dir / "plugin-manifest.yaml").write_text( - textwrap.dedent( - """ - description: "Demo plugin" - author: "ContextForge Team" - version: "0.0.1" - kind: "cpex_demo_plugin.demo_plugin.DemoPluginPlugin" garbage - available_hooks: - - "tool_pre_invoke" - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("trailing content", result.stderr.lower()) - - def test_validator_rejects_missing_plugin_entry_point(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("project.entry-points", result.stderr.lower()) - - def test_validator_rejects_nondict_project_entry_points_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - entry-points = [] - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("entry-points", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_nondict_cpex_plugins_entry_points_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [project.entry-points] - cpex.plugins = [] - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("cpex.plugins", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_nonstring_sibling_cpex_plugins_entry_point_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [project.entry-points."cpex.plugins"] - demo_plugin = "cpex_demo_plugin.demo_plugin:DemoPluginPlugin" - other = 123 - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("cpex.plugins", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_missing_slug_specific_plugin_entry_point(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [project.entry-points."cpex.plugins"] - other_plugin = "cpex_demo_plugin.demo_plugin:DemoPluginPlugin" - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("entry point", result.stderr.lower()) - - def test_validator_rejects_mismatched_manifest_kind_and_entry_point(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - package_dir = plugin_dir / "cpex_demo_plugin" - (package_dir / "plugin-manifest.yaml").write_text( - textwrap.dedent( - """ - description: "Demo plugin" - author: "ContextForge Team" - version: "0.0.1" - kind: "cpex_demo_plugin.other.OtherPlugin" - available_hooks: - - "tool_pre_invoke" - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("kind mismatch", result.stderr.lower()) - - def test_validator_rejects_noncanonical_manifest_kind_separator(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - package_dir = plugin_dir / "cpex_demo_plugin" - (package_dir / "plugin-manifest.yaml").write_text( - textwrap.dedent( - """ - description: "Demo plugin" - author: "ContextForge Team" - version: "0.0.1" - kind: "cpex_demo_plugin.demo_plugin:DemoPluginPlugin" - available_hooks: - - "tool_pre_invoke" - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("module.object", result.stderr.lower()) - - def test_validator_rejects_equal_noncanonical_kind_strings(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - package_dir = plugin_dir / "cpex_demo_plugin" - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [project.entry-points."cpex.plugins"] - demo_plugin = "cpex_demo_plugin.demo_plugin:DemoPluginPlugin" - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - (package_dir / "plugin-manifest.yaml").write_text( - textwrap.dedent( - """ - description: "Demo plugin" - author: "ContextForge Team" - version: "0.0.1" - kind: "cpex_demo_plugin.demo_plugin:DemoPluginPlugin" - available_hooks: - - "tool_pre_invoke" - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("module.object", result.stderr.lower()) - - def test_validator_rejects_noncanonical_entry_point_with_canonical_manifest(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [project.entry-points."cpex.plugins"] - demo_plugin = "cpex_demo_plugin.demo_plugin.DemoPluginPlugin" - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("module:object", result.stderr.lower()) - - def test_validator_rejects_malformed_entry_point_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [project.entry-points."cpex.plugins"] - demo_plugin = "cpex_demo_plugin.demo_plugin:" - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("module:object", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_nested_object_kind_reference(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - package_dir = plugin_dir / "cpex_demo_plugin" - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [project.entry-points."cpex.plugins"] - demo_plugin = "cpex_demo_plugin.demo_plugin:DemoPluginPlugin" - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - (package_dir / "plugin-manifest.yaml").write_text( - textwrap.dedent( - """ - description: "Demo plugin" - author: "ContextForge Team" - version: "0.0.1" - kind: "cpex_demo_plugin.demo_plugin.DemoPluginPlugin.nested" - available_hooks: - - "tool_pre_invoke" - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("kind mismatch", result.stderr.lower()) - - def test_validator_rejects_entry_point_with_extras(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - package_dir = plugin_dir / "cpex_demo_plugin" - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [project.entry-points."cpex.plugins"] - demo_plugin = "cpex_demo_plugin.demo_plugin:DemoPluginPlugin[extra]" - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - (package_dir / "plugin-manifest.yaml").write_text( - textwrap.dedent( - """ - description: "Demo plugin" - author: "ContextForge Team" - version: "0.0.1" - kind: "cpex_demo_plugin.demo_plugin.DemoPluginPlugin[extra]" - available_hooks: - - "tool_pre_invoke" - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("module.object", result.stderr.lower()) - - def test_validator_rejects_whitespace_padded_kind_reference(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - package_dir = plugin_dir / "cpex_demo_plugin" - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [project.entry-points."cpex.plugins"] - demo_plugin = " cpex_demo_plugin.demo_plugin:DemoPluginPlugin " - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - (package_dir / "plugin-manifest.yaml").write_text( - textwrap.dedent( - """ - description: "Demo plugin" - author: "ContextForge Team" - version: "0.0.1" - kind: " cpex_demo_plugin.demo_plugin.DemoPluginPlugin " - available_hooks: - - "tool_pre_invoke" - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("module.object", result.stderr.lower()) - - def test_validator_rejects_nondict_project_table_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - project = [] - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("project", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_nondict_cargo_package_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "Cargo.toml").write_text("package = []\n") - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("cargo.toml", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_malformed_workspace_cargo_toml_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "plugins" / "rust" / "python-package").mkdir(parents=True) - (root / "Cargo.toml").write_text("[workspace\nmembers = []\n") - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("invalid toml", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_nondict_workspace_table_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "plugins" / "rust" / "python-package").mkdir(parents=True) - (root / "Cargo.toml").write_text("workspace = []\n") - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("workspace", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_nonlist_workspace_members_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "plugins" / "rust" / "python-package").mkdir(parents=True) - (root / "Cargo.toml").write_text("[workspace]\nmembers = \"demo_plugin\"\n") - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("workspace", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_nonstr_workspace_member_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "plugins" / "rust" / "python-package").mkdir(parents=True) - (root / "Cargo.toml").write_text('[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin", 123]\n') - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("workspace", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_nondict_workspace_package_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "plugins" / "rust" / "python-package").mkdir(parents=True) - (root / "Cargo.toml").write_text("[workspace]\nmembers = []\npackage = []\n") - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("workspace.package", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_nondict_project_dynamic_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = "version" - - [project.entry-points."cpex.plugins"] - demo_plugin = "cpex_demo_plugin.demo_plugin:DemoPluginPlugin" - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "." - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("dynamic version", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_nondict_tool_maturin_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [project.entry-points."cpex.plugins"] - demo_plugin = "cpex_demo_plugin.demo_plugin:DemoPluginPlugin" - - [tool] - maturin = [] - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("tool.maturin", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_malformed_pyproject_toml_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text("[project\nname = 'broken'\n") - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("invalid toml", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_validator_rejects_malformed_cargo_toml_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "Cargo.toml").write_text("[package\nname = 'broken'\n") - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("invalid toml", result.stderr.lower()) - self.assertNotIn("traceback", result.stderr.lower()) - - def test_pii_manifest_defaults_match_runtime_defaults(self) -> None: - manifest_defaults = self._parse_manifest_defaults( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "pii_filter" - / "cpex_pii_filter" - / "plugin-manifest.yaml" - ) - runtime_defaults = self._extract_pii_runtime_defaults() - self.assertEqual(manifest_defaults, runtime_defaults) - - def test_validator_rejects_manifest_version_mismatch(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - package_dir = plugin_dir / "cpex_demo_plugin" - (plugin_dir / "Cargo.toml").write_text( - textwrap.dedent( - """ - [package] - name = "demo_plugin" - version = "1.2.3" - repository = "https://github.com/IBM/cpex-plugins" - """ - ).strip() - + "\n" - ) - (package_dir / "plugin-manifest.yaml").write_text( - textwrap.dedent( - """ - description: "Demo plugin" - author: "ContextForge Team" - version: "9.9.9" - available_hooks: - - "tool_pre_invoke" - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("version mismatch", result.stderr.lower()) - - def test_validator_accepts_manifest_version_with_yaml_comments(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - package_dir = plugin_dir / "cpex_demo_plugin" - (plugin_dir / "Cargo.toml").write_text( - textwrap.dedent( - """ - [package] - name = "demo_plugin" - version = "1.2.3" - repository.workspace = true - """ - ).strip() - + "\n" - ) - (package_dir / "plugin-manifest.yaml").write_text( - textwrap.dedent( - """ - description: "Demo plugin" - author: "ContextForge Team" - version: "1.2.3" # inline comment - kind: "cpex_demo_plugin.demo_plugin.DemoPluginPlugin" - available_hooks: - - "tool_pre_invoke" - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertEqual(result.returncode, 0, result.stderr) - - def test_validator_rejects_plugin_missing_from_workspace(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text('[workspace]\nmembers = []\n') - self._create_plugin(root, "demo_plugin") - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("workspace", result.stderr.lower()) - - def test_validator_allows_extra_workspace_members(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin", "plugins/rust/python-package/stale_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - self._create_plugin(root, "demo_plugin") - - result = run_catalog("validate", str(root)) - self.assertEqual(result.returncode, 0, result.stderr) - - def test_validator_ignores_plugin_manifests_outside_managed_root(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n', - ) - self._create_plugin(root, "demo_plugin") - stray_dir = root / "legacy_plugin" - stray_package = stray_dir / "cpex_legacy_plugin" - stray_package.mkdir(parents=True) - (stray_package / "__init__.py").write_text("") - (stray_package / "plugin-manifest.yaml").write_text( - 'description: "Legacy plugin"\nauthor: "ContextForge Team"\nversion: "0.0.1"\navailable_hooks:\n - "tool_pre_invoke"\n' - ) - - result = run_catalog("validate", str(root)) - self.assertEqual(result.returncode, 0, result.stderr) - - def test_changed_returns_plugins_for_git_diff(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter"]\n', - ) - - readme = root / "plugins" / "rust" / "python-package" / "rate_limiter" / "README.md" - readme.parent.mkdir(parents=True, exist_ok=True) - readme.write_text("# Rate limiter\n") - cargo = readme.parent / "Cargo.toml" - cargo.write_text( - "[package]\nname = \"rate_limiter\"\nversion = \"0.0.1\"\nrepository = \"https://github.com/IBM/cpex-plugins\"\n" - ) - pyproject = readme.parent / "pyproject.toml" - pyproject.write_text( - "[project]\nname = \"cpex-rate-limiter\"\ndynamic = [\"version\"]\n\n" - "[project.entry-points.\"cpex.plugins\"]\n" - "rate_limiter = \"cpex_rate_limiter.rate_limiter:RateLimiterPlugin\"\n\n" - "[tool.maturin]\nmodule-name = \"cpex_rate_limiter.rate_limiter_rust\"\npython-source = \".\"\n" - ) - package_dir = readme.parent / "cpex_rate_limiter" - package_dir.mkdir() - (package_dir / "__init__.py").write_text("") - (package_dir / "plugin-manifest.yaml").write_text( - 'description: "Rate limiter"\nauthor: "ContextForge Team"\nversion: "0.0.1"\nkind: "cpex_rate_limiter.rate_limiter.RateLimiterPlugin"\navailable_hooks:\n - "tool_pre_invoke"\n' - ) - (readme.parent / "Makefile").write_text("all:\n\t@true\n") - (readme.parent / "tests").mkdir() - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - readme.write_text("# Rate limiter\n\nUpdated.\n") - git("add", ".") - git("commit", "--no-verify", "-m", "update readme") - - result = run_catalog("changed", str(root), base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - - payload = json.loads(result.stdout) - self.assertEqual(payload["plugins"], ["rate_limiter"]) - - def test_changed_skips_plugin_ci_for_repo_docs_change(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - (root / "README.md").write_text("# Repo\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (root / "README.md").write_text("# Repo\n\nUpdated docs.\n") - git("add", ".") - git("commit", "--no-verify", "-m", "update docs") - - result = run_catalog("changed", str(root), base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["plugins"], ["pii_filter", "rate_limiter"]) - - def test_changed_returns_only_modified_plugin_in_multi_plugin_repo(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - ( - root - / "plugins" - / "rust" - / "python-package" - / "pii_filter" - / "README.md" - ).write_text("# pii_filter\n\nUpdated.\n") - git("add", ".") - git("commit", "--no-verify", "-m", "update pii filter readme") - - result = run_catalog("changed", str(root), base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["plugins"], ["pii_filter"]) - - def test_changed_skips_plugin_ci_for_root_makefile_change(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - (root / "Makefile").write_text("help:\n\t@true\n") - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (root / "Makefile").write_text("help:\n\t@echo plugins-list\n") - git("add", ".") - git("commit", "--no-verify", "-m", "update root makefile") - - result = run_catalog("changed", str(root), base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["plugins"], ["pii_filter", "rate_limiter"]) - - def test_changed_returns_all_plugins_for_workflow_change(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - workflow_dir = root / ".github" / "workflows" - workflow_dir.mkdir(parents=True) - workflow = workflow_dir / "ci-rust-python-package.yaml" - workflow.write_text("name: ci\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - workflow.write_text("name: ci\non: pull_request\n") - git("add", ".") - git("commit", "--no-verify", "-m", "update workflow") - - result = run_catalog("changed", str(root), base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["plugins"], ["pii_filter", "rate_limiter"]) - - def test_release_info_accepts_canonical_tag(self) -> None: - cargo = tomllib.loads( - ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "rate_limiter" - / "Cargo.toml" - ).read_text() - ) - tag = f"rate-limiter-v{cargo['package']['version']}" - result = run_catalog("release-info", str(REPO_ROOT), tag) - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["slug"], "rate_limiter") - self.assertEqual( - payload["kind"], - "cpex_rate_limiter.rate_limiter.RateLimiterPlugin", - ) - self.assertEqual( - payload["release_wheel_matrix"], - [ - {"runner": "ubuntu-latest", "platform": "linux-x86_64"}, - {"runner": "ubuntu-24.04-arm", "platform": "linux-aarch64"}, - {"runner": "ubuntu-24.04-s390x", "platform": "linux-s390x"}, - {"runner": "ubuntu-24.04-ppc64le", "platform": "linux-ppc64le"}, - {"runner": "macos-latest", "platform": "macos-arm64"}, - {"runner": "windows-latest", "platform": "windows-x86_64"}, - ], - ) - - def test_release_info_gives_pii_filter_the_same_target_matrix(self) -> None: - cargo = tomllib.loads( - ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "pii_filter" - / "Cargo.toml" - ).read_text() - ) - tag = f"pii-filter-v{cargo['package']['version']}" - result = run_catalog("release-info", str(REPO_ROOT), tag) - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["slug"], "pii_filter") - self.assertEqual( - payload["release_wheel_matrix"], - [ - {"runner": "ubuntu-latest", "platform": "linux-x86_64"}, - {"runner": "ubuntu-24.04-arm", "platform": "linux-aarch64"}, - {"runner": "ubuntu-24.04-s390x", "platform": "linux-s390x"}, - {"runner": "ubuntu-24.04-ppc64le", "platform": "linux-ppc64le"}, - {"runner": "macos-latest", "platform": "macos-arm64"}, - {"runner": "windows-latest", "platform": "windows-x86_64"}, - ], - ) - - def test_release_info_rejects_noncanonical_tag(self) -> None: - result = run_catalog("release-info", str(REPO_ROOT), "rate_limiter-v0.1.0") - self.assertNotEqual(result.returncode, 0) - self.assertIn("canonical", result.stderr.lower()) - - def test_release_info_field_supports_kind(self) -> None: - cargo = tomllib.loads( - ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "pii_filter" - / "Cargo.toml" - ).read_text() - ) - tag = f"pii-filter-v{cargo['package']['version']}" - result = run_catalog("release-info-field", str(REPO_ROOT), tag, "kind") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "cpex_pii_filter.pii_filter.PIIFilterPlugin") - - result = run_catalog("release-info-field", str(REPO_ROOT), tag, "language") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "rust") - - def test_ci_selection_returns_has_plugins_contract(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (root / "README.md").write_text("# Repo docs only\n") - git("add", ".") - git("commit", "--no-verify", "-m", "docs only") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self._assert_payload_contains(payload, { - "plugins": ["pii_filter", "rate_limiter"], - "has_plugins": True, - "plugin_count": 2, - "cargo_packages": ["pii_filter", "rate_limiter"], - "mutation_cargo_packages": [], - "has_mutation_cargo_packages": False, - "mutation_jobs": [], - "release_validation_tags": [], - "has_release_validation_tags": False, - }) - - def test_ci_selection_detects_plugin_version_bump_for_release_validation(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - rate_limiter = self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (rate_limiter / "Cargo.toml").write_text( - '[package]\nname = "rate_limiter"\nversion = "0.0.2"\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - (rate_limiter / "cpex_rate_limiter" / "plugin-manifest.yaml").write_text( - 'description: "rate_limiter"\nauthor: "ContextForge Team"\nversion: "0.0.2"\nkind: "cpex_rate_limiter.rate_limiter.RateLimiterPlugin"\navailable_hooks:\n - "tool_pre_invoke"\n' - ) - git("add", ".") - git("commit", "--no-verify", "-m", "bump rate limiter") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["release_validation_tags"], ["rate-limiter-v0.0.2"]) - self.assertTrue(payload["has_release_validation_tags"]) - - def test_ci_selection_detects_new_plugin_for_initial_release(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter"]\n' - ) - self._create_plugin(root, "rate_limiter") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (root / "Cargo.toml").write_text( - "[workspace]\nmembers = [\n" - ' "plugins/rust/python-package/rate_limiter",\n' - ' "plugins/rust/python-package/sql_sanitizer",\n' - "]\n" - ) - self._create_plugin(root, "sql_sanitizer") - git("add", ".") - git("commit", "--no-verify", "-m", "add sql sanitizer") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["release_validation_tags"], ["sql-sanitizer-v0.0.1"]) - self.assertTrue(payload["has_release_validation_tags"]) - - def test_ci_selection_treats_catalog_test_change_as_not_shared(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - tests_dir = root / "tests" - tests_dir.mkdir() - (tests_dir / "test_plugin_catalog.py").write_text("# shared test change\n") - git("add", ".") - git("commit", "--no-verify", "-m", "catalog test change") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self._assert_payload_contains(payload, { - "plugins": [], - "has_plugins": False, - "plugin_count": 0, - "cargo_packages": [], - "mutation_cargo_packages": [], - "has_mutation_cargo_packages": False, - "mutation_jobs": [], - "release_validation_tags": [], - "has_release_validation_tags": False, - }) - - def test_ci_selection_treats_shared_tool_changes_as_all_plugins(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - tools_dir = root / "tools" - tools_dir.mkdir() - (tools_dir / "plugin_catalog.py").write_text("# shared tool change\n") - git("add", ".") - git("commit", "--no-verify", "-m", "shared tool input") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self._assert_payload_contains(payload, { - "plugins": ["pii_filter", "rate_limiter"], - "has_plugins": True, - "plugin_count": 2, - "cargo_packages": ["pii_filter", "rate_limiter"], - "mutation_cargo_packages": [], - "has_mutation_cargo_packages": False, - "mutation_jobs": [], - "release_validation_tags": [], - "has_release_validation_tags": False, - }) - - def test_ci_selection_treats_tooling_config_changes_as_all_plugins(self) -> None: - for config_path in (".cargo/mutants.toml", ".config/nextest.toml"): - with self.subTest(config_path=config_path): - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - path = root / config_path - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("# shared tooling config change\n") - git("add", ".") - git("commit", "--no-verify", "-m", "shared tooling config input") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self._assert_payload_contains(payload, { - "plugins": ["pii_filter", "rate_limiter"], - "has_plugins": True, - "plugin_count": 2, - "cargo_packages": ["pii_filter", "rate_limiter"], - "mutation_cargo_packages": [], - "has_mutation_cargo_packages": False, - "mutation_jobs": [], - "release_validation_tags": [], - "has_release_validation_tags": False, - }) - - def test_ci_selection_skips_mutation_for_unrelated_tooling_config(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - path = root / ".config" / "editor.toml" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("# unrelated tooling config change\n") - git("add", ".") - git("commit", "--no-verify", "-m", "unrelated tooling config input") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["plugins"], ["pii_filter", "rate_limiter"]) - self.assertEqual(payload["mutation_cargo_packages"], []) - self.assertEqual(payload["mutation_jobs"], []) - - def test_ci_selection_treats_cargo_lock_change_as_all_plugins(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - (root / "Cargo.lock").write_text("# seed\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (root / "Cargo.lock").write_text("# updated\n") - git("add", ".") - git("commit", "--no-verify", "-m", "lockfile update") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self._assert_payload_contains(payload, { - "plugins": ["pii_filter", "rate_limiter"], - "has_plugins": True, - "plugin_count": 2, - "cargo_packages": ["pii_filter", "rate_limiter"], - "mutation_cargo_packages": [], - "has_mutation_cargo_packages": False, - "mutation_jobs": [], - "release_validation_tags": [], - "has_release_validation_tags": False, - }) - - def test_ci_selection_treats_root_python_workspace_change_as_all_plugins(self) -> None: - for config_path in ("pyproject.toml", "uv.lock"): - with self.subTest(config_path=config_path): - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - (root / config_path).write_text("# seed\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (root / config_path).write_text("# updated\n") - git("add", ".") - git("commit", "--no-verify", "-m", "python workspace update") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["plugins"], ["pii_filter", "rate_limiter"]) - self.assertEqual(payload["cargo_packages"], ["pii_filter", "rate_limiter"]) - self.assertEqual(payload["plugin_count"], 2) - - def test_ci_selection_keeps_cargo_lock_with_plugin_change_plugin_scoped(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - rate_limiter = self._create_plugin(root, "rate_limiter") - pii_filter = self._create_plugin(root, "pii_filter") - (rate_limiter / "src").mkdir() - (pii_filter / "src").mkdir() - (root / "Cargo.lock").write_text("# seed\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (rate_limiter / "src" / "lib.rs").write_text("// rate_limiter update\n") - (root / "Cargo.lock").write_text("# updated\n") - git("add", ".") - git("commit", "--no-verify", "-m", "plugin lockfile update") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["plugins"], ["rate_limiter"]) - self.assertEqual(payload["cargo_packages"], ["rate_limiter"]) - self.assertEqual(payload["plugin_count"], 1) - - base_sha = git("rev-parse", "HEAD").stdout.strip() - (rate_limiter / "src" / "lib.rs").write_text("// rate_limiter update 2\n") - (pii_filter / "src" / "lib.rs").write_text("// pii_filter update\n") - (root / "Cargo.lock").write_text("# updated again\n") - git("add", ".") - git("commit", "--no-verify", "-m", "multi-plugin lockfile update") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["plugins"], ["pii_filter", "rate_limiter"]) - self.assertEqual(payload["cargo_packages"], ["pii_filter", "rate_limiter"]) - self.assertEqual(payload["plugin_count"], 2) - - def test_ci_selection_treats_deny_config_change_as_all_plugins(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - (root / "deny.toml").write_text("[licenses]\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (root / "deny.toml").write_text("[licenses]\nconfidence-threshold = 0.95\n") - git("add", ".") - git("commit", "--no-verify", "-m", "deny policy update") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self._assert_payload_contains(payload, { - "plugins": ["pii_filter", "rate_limiter"], - "has_plugins": True, - "plugin_count": 2, - "cargo_packages": ["pii_filter", "rate_limiter"], - "mutation_cargo_packages": [], - "has_mutation_cargo_packages": False, - "mutation_jobs": [], - "release_validation_tags": [], - "has_release_validation_tags": False, - }) - - def test_changed_returns_plugin_for_plugin_integration_test_change(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - integration_dir = root / "plugins" / "tests" / "pii_filter" - integration_dir.mkdir(parents=True) - (integration_dir / "test_integration.py").write_text("# seed\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (integration_dir / "test_integration.py").write_text("# updated\n") - git("add", ".") - git("commit", "--no-verify", "-m", "plugin integration change") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self._assert_payload_contains(payload, { - "plugins": ["pii_filter"], - "has_plugins": True, - "plugin_count": 1, - "cargo_packages": ["pii_filter"], - "mutation_cargo_packages": [], - "has_mutation_cargo_packages": False, - "mutation_jobs": [], - "release_validation_tags": [], - "has_release_validation_tags": False, - }) - - def test_ci_selection_treats_shared_plugin_tests_change_as_all_plugins(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - shared_tests = root / "plugins" / "tests" - shared_tests.mkdir(parents=True) - (shared_tests / "plugin_hooks.py").write_text("# seed\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (shared_tests / "plugin_hooks.py").write_text("# updated\n") - git("add", ".") - git("commit", "--no-verify", "-m", "shared plugin test harness change") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self._assert_payload_contains(payload, { - "plugins": ["pii_filter", "rate_limiter"], - "has_plugins": True, - "plugin_count": 2, - "cargo_packages": ["pii_filter", "rate_limiter"], - "mutation_cargo_packages": [], - "has_mutation_cargo_packages": False, - "mutation_jobs": [], - "release_validation_tags": [], - "has_release_validation_tags": False, - }) - - def test_ci_selection_treats_shared_crate_changes_as_all_plugins(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - (root / "plugins" / "rust" / "python-package" / "rate_limiter" / "Cargo.toml").write_text( - '[package]\nname = "rate_limiter"\nversion = "0.0.1"\nrepository = "https://github.com/IBM/cpex-plugins"\n\n' - "[dependencies]\ncpex_framework_bridge = { workspace = true }\n" - ) - shared_crate = root / "crates" / "framework_bridge" / "src" - shared_crate.mkdir(parents=True) - (shared_crate / "lib.rs").write_text("// shared crate change\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (shared_crate / "lib.rs").write_text("// shared crate change\n// update\n") - git("add", ".") - git("commit", "--no-verify", "-m", "shared crate change") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self._assert_payload_contains(payload, { - "plugins": ["pii_filter", "rate_limiter"], - "has_plugins": True, - "plugin_count": 2, - "cargo_packages": ["pii_filter", "rate_limiter"], - "mutation_cargo_packages": ["cpex_framework_bridge"], - "has_mutation_cargo_packages": True, - "mutation_jobs": [ - { - "cargo_package": "cpex_framework_bridge", - "in_diff": True, - "test_packages": ["rate_limiter"], - } - ], - "release_validation_tags": [], - "has_release_validation_tags": False, - }) - - def test_ci_selection_ignores_tooling_config_for_shared_crate_mutation_jobs(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - (root / "plugins" / "rust" / "python-package" / "rate_limiter" / "Cargo.toml").write_text( - '[package]\nname = "rate_limiter"\nversion = "0.0.1"\nrepository = "https://github.com/IBM/cpex-plugins"\n\n' - "[dependencies]\ncpex_framework_bridge = { workspace = true }\n" - ) - shared_crate = root / "crates" / "framework_bridge" / "src" - shared_crate.mkdir(parents=True) - (shared_crate / "lib.rs").write_text("// shared crate change\n") - config_path = root / ".cargo" / "mutants.toml" - config_path.parent.mkdir(parents=True) - config_path.write_text("# seed\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (shared_crate / "lib.rs").write_text("// shared crate change\n// update\n") - config_path.write_text("# updated\n") - git("add", ".") - git("commit", "--no-verify", "-m", "mixed mutation inputs") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual( - payload["mutation_jobs"], - [ - { - "cargo_package": "cpex_framework_bridge", - "in_diff": True, - "test_packages": ["rate_limiter"], - }, - ], - ) - - def test_ci_selection_ignores_tooling_config_for_plugin_rust_mutation_jobs(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - ) - rate_limiter = self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - src_dir = rate_limiter / "src" - src_dir.mkdir() - (src_dir / "lib.rs").write_text("pub fn rate_limit() {}\n") - config_path = root / ".cargo" / "mutants.toml" - config_path.parent.mkdir(parents=True) - config_path.write_text("# seed\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (src_dir / "lib.rs").write_text("pub fn rate_limit() -> bool { true }\n") - config_path.write_text("# updated\n") - git("add", ".") - git("commit", "--no-verify", "-m", "mixed plugin and config inputs") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual( - payload["mutation_jobs"], - [ - {"cargo_package": "rate_limiter", "in_diff": True, "test_packages": []}, - ], - ) - - def test_framework_bridge_mutation_dependents_match_real_manifests(self) -> None: - plugin_root = REPO_ROOT / "plugins" / "rust" / "python-package" - dependents = [] - for manifest_path in sorted(plugin_root.glob("*/Cargo.toml")): - with manifest_path.open("rb") as handle: - manifest = tomllib.load(handle) - if "cpex_framework_bridge" in manifest.get("dependencies", {}): - dependents.append(manifest_path.parent.name) - - self.assertEqual( - dependents, - [ - "encoded_exfil_detection", - "output_length_guard", - "pii_filter", - "rate_limiter", - "retry_with_backoff", - "secrets_detection", - "sql_sanitizer", - ], - ) - - def test_framework_bridge_mutation_job_uses_real_dependents(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - subprocess.run( - ["git", "init"], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - subprocess.run( - ["git", "config", "user.name", "Test User"], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - subprocess.run( - ["git", "config", "user.email", "test@example.com"], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - subprocess.run( - ["git", "config", "gc.auto", "0"], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - subprocess.run( - ["git", "config", "maintenance.auto", "false"], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - shutil.copytree(REPO_ROOT / "plugins", root / "plugins") - shutil.copytree(REPO_ROOT / "crates", root / "crates") - (root / "Cargo.toml").write_text((REPO_ROOT / "Cargo.toml").read_text()) - (root / "pyproject.toml").write_text( - (REPO_ROOT / "pyproject.toml").read_text() - ) - subprocess.run( - ["git", "add", "."], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - subprocess.run( - ["git", "commit", "--no-verify", "-m", "seed real manifests"], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - base_sha = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=root, - text=True, - capture_output=True, - check=True, - ).stdout.strip() - - bridge_lib = root / "crates" / "framework_bridge" / "src" / "lib.rs" - bridge_lib.write_text(bridge_lib.read_text() + "\n// mutation route test\n") - subprocess.run( - ["git", "add", "."], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - subprocess.run( - ["git", "commit", "--no-verify", "-m", "shared crate change"], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertIn( - { - "cargo_package": "cpex_framework_bridge", - "in_diff": True, - "test_packages": [ - "encoded_exfil_detection", - "output_length_guard", - "pii_filter", - "rate_limiter", - "retry_with_backoff", - "secrets_detection", - "sql_sanitizer", - ], - }, - payload["mutation_jobs"], - ) - - def test_ci_selection_reports_cargo_packages_for_single_plugin_diff(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - readme = root / "plugins" / "rust" / "python-package" / "rate_limiter" / "README.md" - readme.write_text("# changed\n") - git("add", ".") - git("commit", "--no-verify", "-m", "single plugin change") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self._assert_payload_contains(payload, { - "plugins": ["rate_limiter"], - "has_plugins": True, - "plugin_count": 1, - "cargo_packages": ["rate_limiter"], - "mutation_cargo_packages": [], - "has_mutation_cargo_packages": False, - "mutation_jobs": [], - "release_validation_tags": [], - "has_release_validation_tags": False, - }) - - def test_ci_selection_reports_mutation_package_for_single_rust_diff(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - plugin_dir = self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - src_dir = plugin_dir / "src" - src_dir.mkdir() - (src_dir / "lib.rs").write_text("pub fn rate_limit() {}\n") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - (src_dir / "lib.rs").write_text("pub fn rate_limit() -> bool { true }\n") - git("add", ".") - git("commit", "--no-verify", "-m", "single rust change") - - result = run_catalog("ci-selection", str(root), "diff", base_sha, "HEAD") - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self._assert_payload_contains(payload, { - "plugins": ["rate_limiter"], - "has_plugins": True, - "plugin_count": 1, - "cargo_packages": ["rate_limiter"], - "mutation_cargo_packages": ["rate_limiter"], - "has_mutation_cargo_packages": True, - "mutation_jobs": [ - {"cargo_package": "rate_limiter", "in_diff": True, "test_packages": []} - ], - "release_validation_tags": [], - "has_release_validation_tags": False, - }) - - def test_ci_selection_field_prints_json_and_bool_scalars(self) -> None: - expected_rust_plugins = [ - "encoded_exfil_detection", - "output_length_guard", - "pii_filter", - "rate_limiter", - "retry_with_backoff", - "secrets_detection", - "sql_sanitizer", - "url_reputation", - ] - expected_plugins = [ - "encoded_exfil_detection", - "ica_metering_exporter", - *expected_rust_plugins[1:], - ] - result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "plugins") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(json.loads(result.stdout), expected_plugins) - - result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "has_plugins") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "true") - - result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "plugin_count") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "9") - - result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "cargo_packages") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(json.loads(result.stdout), expected_rust_plugins) - - result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "mutation_cargo_packages") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(json.loads(result.stdout), expected_rust_plugins) - - result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "mutation_jobs") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual( - json.loads(result.stdout), - [ - {"cargo_package": plugin, "in_diff": False, "test_packages": []} - for plugin in expected_rust_plugins - ], - ) - - result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "has_mutation_cargo_packages") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "true") - - result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "release_validation_tags") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(json.loads(result.stdout), []) +# Copyright 2026 +# SPDX-License-Identifier: Apache-2.0 +"""Catalog behavior at discovery, CI selection, release, coverage, and CLI boundaries.""" - result = run_catalog("ci-selection-field", str(REPO_ROOT), "all", "", "", "has_release_validation_tags") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "false") - - expected_split_fields = { - "rust_plugins": json.dumps(expected_rust_plugins), - "python_plugins": '["ica_metering_exporter"]', - "has_rust_plugins": "true", - "has_python_plugins": "true", - "rust_plugin_count": "8", - "python_plugin_count": "1", - "rust_release_validation_tags": "[]", - "python_release_validation_tags": "[]", - "has_rust_release_validation_tags": "false", - "has_python_release_validation_tags": "false", - } - for field, expected in expected_split_fields.items(): - with self.subTest(field=field): - result = run_catalog( - "ci-selection-field", str(REPO_ROOT), "all", "", "", field - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), expected) - - def test_ci_selection_field_supports_diff_mode(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - git = lambda *args: subprocess.run( # noqa: E731 - ["git", *args], - cwd=root, - text=True, - capture_output=True, - check=True, - ) - git("init") - git("config", "user.name", "Test User") - git("config", "user.email", "test@example.com") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/rate_limiter", "plugins/rust/python-package/pii_filter"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "rate_limiter") - self._create_plugin(root, "pii_filter") - git("add", ".") - git("commit", "--no-verify", "-m", "seed layout") - base_sha = git("rev-parse", "HEAD").stdout.strip() - - makefile = root / "Makefile" - makefile.write_text("help:\n\t@echo plugins-list\n") - git("add", ".") - git("commit", "--no-verify", "-m", "root makefile change") - - result = run_catalog( - "ci-selection-field", - str(root), - "diff", - base_sha, - "HEAD", - "plugins", - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(json.loads(result.stdout), ["pii_filter", "rate_limiter"]) - - result = run_catalog( - "ci-selection-field", - str(root), - "diff", - base_sha, - "HEAD", - "plugin_count", - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "2") - - def test_ci_workflow_shared_paths_match_catalog_contract(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-rust-python-package.yaml" - ).read_text() - expected_paths = { - "Makefile", - "Cargo.toml", - "Cargo.lock", - ".cargo/**", - ".config/nextest.toml", - "deny.toml", - "crates/**", - "README.md", - "DEVELOPING.md", - "TESTING.md", - "plugins/tests/**", - "tools/**", - ".github/workflows/ci-rust-python-package.yaml", - ".github/workflows/release-rust-python-package.yaml", - } - actual_paths = { - match.group(1) - for match in re.finditer(r'- "([^"]+)"', workflow) - } - self.assertTrue(expected_paths.issubset(actual_paths)) - - def test_ci_workflow_only_cancels_superseded_pull_requests(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-rust-python-package.yaml" - ).read_text() - self.assertIn("concurrency:", workflow) - self.assertIn("github.head_ref || github.ref_name", workflow) - self.assertIn( - "cancel-in-progress: ${{ github.event_name == 'pull_request' }}", - workflow, - ) - - def test_catalog_workflow_paths_match_contract(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-plugin-catalog.yaml" - ).read_text() - expected_paths = { - "tests/test_plugin_catalog.py", - "tools/plugin_catalog.py", - ".github/workflows/ci-plugin-catalog.yaml", - ".github/workflows/ci-rust-python-package.yaml", - ".github/workflows/release-rust-python-package.yaml", - ".github/workflows/ci-python-package.yaml", - ".github/workflows/release-python-package.yaml", - "plugins/python/**", - "tools/validate_ci_selection.py", - "pyproject.toml", - } - actual_paths = { - match.group(1) - for match in re.finditer(r'- "([^"]+)"', workflow) - } - self.assertTrue(expected_paths.issubset(actual_paths)) - - def test_install_wheel_workflow_paths_match_contract(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-install-built-wheel.yaml" - ).read_text() - expected_paths = { - "tests/test_install_built_wheel.py", - "plugins/rust/python-package/**", - ".github/workflows/ci-install-built-wheel.yaml", - ".github/workflows/release-rust-python-package.yaml", - } - actual_paths = { - match.group(1) - for match in re.finditer(r'- "([^"]+)"', workflow) - } - self.assertTrue(expected_paths.issubset(actual_paths)) - - def test_validator_rejects_maturin_module_name_drift(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [tool.maturin] - module-name = "wrong.module" - python-source = "." - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("module-name", result.stderr) - - def test_validator_rejects_maturin_python_source_drift(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/demo_plugin"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n', - ) - plugin_dir = self._create_plugin(root, "demo_plugin") - (plugin_dir / "pyproject.toml").write_text( - textwrap.dedent( - """ - [project] - name = "cpex-demo-plugin" - dynamic = ["version"] - - [tool.maturin] - module-name = "cpex_demo_plugin.demo_plugin_rust" - python-source = "python" - """ - ).strip() - + "\n" - ) - - result = run_catalog("validate", str(root)) - self.assertNotEqual(result.returncode, 0) - self.assertIn("python-source", result.stderr) - - def test_plugin_makefiles_expose_ci_targets(self) -> None: - for slug in ( - "encoded_exfil_detection", - "pii_filter", - "rate_limiter", - "retry_with_backoff", - "secrets_detection", - "url_reputation", - ): - makefile = ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / slug - / "Makefile" - ) - text = makefile.read_text() - self.assertRegex(text, r"(?m)^\.PHONY:.*\binstall-wheel\b") - self.assertRegex(text, r"(?m)^install-wheel:") - self.assertRegex(text, r"(?m)^\.PHONY:.*\bci\b") - self.assertRegex(text, r"(?m)^\.PHONY:.*\bci-build\b") - self.assertRegex(text, r"(?m)^ci-build:.*\binstall-wheel\b") - self.assertRegex(text, r"(?m)^ci:") - self.assertNotRegex(text, r"(?m)^ci:.*(?:^|\s)install(?:\s|$)") - self.assertRegex(text, r"(?m)^ci:.*\btest-integration\b") - - def test_existing_benchmark_plugins_keep_bench_targets(self) -> None: - for slug in ("pii_filter", "rate_limiter"): - makefile = ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / slug - / "Makefile" - ) - text = makefile.read_text() - self.assertRegex(text, r"(?m)^\.PHONY:.*\bbench-no-run\b") - self.assertRegex(text, r"(?m)^bench-no-run:") - - def test_ci_workflow_uses_make_targets_for_plugin_checks(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-rust-python-package.yaml" - ).read_text() - self.assertIn("push:\n branches: [main]", workflow) - self.assertIn("pull_request:\n branches: [main]", workflow) - self.assertIn('- "Makefile"', workflow) - self.assertNotIn("pulls?state=open&head=", workflow) - self.assertNotIn("dedupe:", workflow) - self.assertIn("if: needs.validate-and-detect.outputs.has_rust_plugins == 'true'", workflow) - self.assertNotIn("tests/test_plugin_catalog.py", workflow) - self.assertNotIn("tests/test_install_built_wheel.py", workflow) - self.assertIn("python3 tools/plugin_catalog.py ci-selection . diff", workflow) - self.assertIn("run: make ci-build", workflow) - self.assertIn("run: make ci", workflow) - self.assertIn("shell: bash", workflow) - self.assertIn("rustc --version", workflow) - self.assertIn("working-directory: plugins/rust/python-package/${{ matrix.plugin }}", workflow) - self.assertIn("release-validation:", workflow) - self.assertIn("uses: ./.github/workflows/release-rust-python-package.yaml", workflow) - self.assertIn("tag: ${{ matrix.tag }}", workflow) - self.assertIn("repository: testpypi", workflow) - self.assertIn("publish_enabled: false", workflow) - self.assertNotIn("tools/plugin_catalog.py ci-selection-field", workflow) - self.assertNotIn("tools/plugin_catalog.py changed", workflow) - self.assertNotIn("tools/plugin_catalog.py list", workflow) - self.assertNotIn("dtolnay/rust-toolchain", workflow) - self.assertNotRegex(workflow, r"run:\s*uv run pytest") - self.assertNotRegex(workflow, r"run:\s*uv run maturin develop") - self.assertNotIn("python3 - <<'PY'", workflow) - self.assertIn("uv==0.9.30", workflow) - self.assertIn("maturin==1.12.6", workflow) - self.assertIn( - "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1", - workflow, - ) - self.assertIn( - "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0", - workflow, - ) - self.assertNotIn("actions/checkout@v4", workflow) - self.assertNotIn("actions/setup-python@v5", workflow) - self.assertNotRegex( - workflow, - r"defaults:\n\s+run:\n\s+shell: bash\n\s+working-directory: .*\$\{\{", - ) +import json +import subprocess +import sys +import tempfile +import unittest +from dataclasses import asdict +from pathlib import Path +from unittest.mock import patch - def test_ci_workflows_route_complete_language_split_contracts(self) -> None: - rust_workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-rust-python-package.yaml" - ).read_text() - python_workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-python-package.yaml" - ).read_text() +from tools import plugin_catalog as catalog - self._assert_workflow_needs_outputs_declared( - rust_workflow, "validate-and-detect" - ) - self._assert_workflow_needs_outputs_declared( - python_workflow, "validate-and-detect" +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = REPO_ROOT / "tools" / "plugin_catalog.py" +SELECTION_VALIDATOR = REPO_ROOT / "tools" / "validate_ci_selection.py" +RUST_PATH = "plugins/rust/python-package/rust_demo" +OTHER_PATH = "plugins/rust/python-package/other_demo" +PYTHON_PATH = "plugins/python/python_demo" +ALL_PLUGINS = ["other_demo", "python_demo", "rust_demo"] + + +class CatalogTestCase(unittest.TestCase): + def setUp(self) -> None: + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + (self.root / "Cargo.toml").write_text(f'[workspace]\nmembers = ["{RUST_PATH}", "{OTHER_PATH}"]\n') + (self.root / "pyproject.toml").write_text(f'[tool.uv.workspace]\nmembers = ["{PYTHON_PATH}"]\n') + self.create_plugin("rust_demo", "rust") + self.create_plugin("other_demo", "rust") + self.create_plugin("python_demo", "python") + + def create_plugin(self, slug: str, language: str) -> Path: + parent = "plugins/python" if language == "python" else "plugins/rust/python-package" + plugin = self.root / parent / slug + module = plugin / f"cpex_{slug}" + module.mkdir(parents=True) + version = 'version = "1.0.0"' if language == "python" else 'dynamic = ["version"]' + pyproject = f'[project]\nname = "cpex-{slug.replace("_", "-")}"\n{version}\n[project.entry-points."cpex.plugins"]\n{slug} = "cpex_{slug}.plugin:DemoPlugin"\n' + if language == "rust": + pyproject += f'[tool.maturin]\nmodule-name = "cpex_{slug}.{slug}_rust"\npython-source = "."\n' + (plugin / "Cargo.toml").write_text(f'[package]\nname = "{slug}"\nversion = "1.0.0"\nrepository = "https://github.com/IBM/cpex-plugins"\n') + (plugin / "pyproject.toml").write_text(pyproject) + (plugin / "Makefile").touch() + (plugin / "README.md").touch() + (module / "__init__.py").touch() + (module / "plugin-manifest.yaml").write_text(f'version: "1.0.0"\nkind: "cpex_{slug}.plugin.DemoPlugin"\n') + return plugin + + def run_catalog(self, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), args[0], str(self.root), *args[1:]], + text=True, + capture_output=True, + check=False, ) - for field in ( - "rust_plugins", - "has_rust_plugins", - "rust_plugin_count", - "rust_release_validation_tags", - "has_rust_release_validation_tags", - ): - self.assertIn(f"echo \"{field}=", rust_workflow) - self.assertIn( - f"{field}: ${{{{ steps.detect.outputs.{field} }}}}", rust_workflow - ) - for field in ( - "python_plugins", - "has_python_plugins", - "python_plugin_count", - "python_release_validation_tags", - "has_python_release_validation_tags", - ): - self.assertIn(f"echo \"{field}=", python_workflow) - self.assertIn( - f"{field}: ${{{{ steps.detect.outputs.{field} }}}}", python_workflow - ) - self.assertNotIn('- "plugins/python/**"', rust_workflow) - self.assertIn("fromJson(needs.validate-and-detect.outputs.rust_plugins)", rust_workflow) - self.assertIn("fromJson(needs.validate-and-detect.outputs.python_plugins)", python_workflow) - self.assertIn( - "working-directory: plugins/python/${{ matrix.plugin }}", python_workflow - ) + def selection(self, paths: list[str]) -> dict: + # Control only the Git boundary; discovery and routing use real fixture files. + with patch.object(catalog, "_git_changed_paths", return_value=paths): + return catalog.ci_selection(self.root, "diff", "base", "head") - def test_python_ci_release_validation_and_tag_creation_match_contract(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-python-package.yaml" - ).read_text() - expected_paths = { - "Makefile", - "pyproject.toml", - "uv.lock", - "plugins/python/**", - "plugins/tests/**", - "tools/**", - "tests/**", - ".github/workflows/ci-python-package.yaml", - ".github/workflows/release-python-package.yaml", - } - for path in expected_paths: - self.assertEqual(workflow.count(f'- "{path}"'), 2) + def assert_invalid_edit(self, file: Path, content: str, error: str) -> None: + original = file.read_text() + file.write_text(content) + try: + with self.assertRaisesRegex(catalog.CatalogError, error): + catalog.discover_plugins(self.root) + finally: + file.write_text(original) - release_validation = self._extract_workflow_job_section( - workflow, "release-validation" - ) - create_tags = self._extract_workflow_job_section( - workflow, "create-release-tags" - ) - self.assertIn("uses: ./.github/workflows/release-python-package.yaml", release_validation) - self.assertIn("has_python_release_validation_tags == 'true'", release_validation) - self.assertIn( - "fromJson(needs.validate-and-detect.outputs.python_release_validation_tags)", - release_validation, - ) - self.assertIn("repository: testpypi", release_validation) - self.assertIn("publish_enabled: false", release_validation) - for dependency in ("validate-and-detect", "build-test", "release-validation"): - self.assertIn(f" - {dependency}", create_tags) - for rust_only_job in ( - "security-policy", - "mutation-testing", - "coverage", - "documentation", - ): - self.assertNotIn(rust_only_job, create_tags) - self.assertIn("always()", create_tags) - self.assertIn("github.event_name == 'push'", create_tags) - self.assertIn("github.ref == 'refs/heads/main'", create_tags) - self.assertIn("has_python_release_validation_tags == 'true'", create_tags) - self.assertIn("needs.build-test.result == 'success'", create_tags) - self.assertIn("needs.release-validation.result == 'skipped'", create_tags) - self.assertIn('git tag "${tag}" "${GITHUB_SHA}"', create_tags) - self.assertIn('git push origin "refs/tags/${tag}"', create_tags) - self.assertIn("gh workflow run release-python-package.yaml", create_tags) - def test_release_workflows_guard_every_post_resolve_job(self) -> None: - workflows = { - "rust": ( - REPO_ROOT - / ".github" - / "workflows" - / "release-rust-python-package.yaml" - ).read_text(), - "python": ( - REPO_ROOT / ".github" / "workflows" / "release-python-package.yaml" - ).read_text(), - } - jobs = { - "rust": ("preflight", "build-wheel", "build-sdist", "publish"), - "python": ("build", "test-built-wheel", "test-built-sdist", "publish"), - } - for language, workflow in workflows.items(): +class DiscoveryTests(CatalogTestCase): + def test_discovers_typed_records_from_both_managed_roots(self) -> None: + records = {record.slug: record for record in catalog.discover_plugins(self.root)} + self.assertEqual(set(records), set(ALL_PLUGINS)) + for slug, language, path in (("rust_demo", "rust", RUST_PATH), ("python_demo", "python", PYTHON_PATH)): with self.subTest(language=language): - self._assert_workflow_needs_outputs_declared(workflow, "resolve") - resolve_section = self._extract_workflow_job_section(workflow, "resolve") - self.assertIn("skip: ${{ steps.resolve.outputs.skip }}", resolve_section) - self.assertIn(f'"${{language}}" != "{language}"', resolve_section) - self.assertIn('echo "skip=true"', resolve_section) - self.assertIn('echo "skip=false"', resolve_section) - for job_name in jobs[language]: - job_section = self._extract_workflow_job_section(workflow, job_name) - self.assertIn("needs.resolve.outputs.skip != 'true'", job_section) - - def test_python_release_artifact_handoff_and_publish_topology(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "release-python-package.yaml" - ).read_text() - build = self._extract_workflow_job_section(workflow, "build") - wheel = self._extract_workflow_job_section(workflow, "test-built-wheel") - sdist = self._extract_workflow_job_section(workflow, "test-built-sdist") - publish = self._extract_workflow_job_section(workflow, "publish") - - self.assertIn("push:\n tags:\n - \"*-v*\"", workflow) - self.assertIn("workflow_call:", workflow) - self.assertIn("workflow_dispatch:", workflow) - self.assertIn("run: uv build --out-dir dist", build) - self.assertIn("name: python-dist", build) - self.assertIn("path: ${{ needs.resolve.outputs.plugin_path }}/dist/", build) - self.assertIn("if-no-files-found: error", build) - for section, artifact_glob in ( - (wheel, "dist/*.whl"), - (sdist, "dist/*.tar.gz"), - ): - self.assertIn("needs: [resolve, build]", section) - self.assertIn("ref: ${{ needs.resolve.outputs.checkout_ref }}", section) - self.assertIn( - "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1", - section, - ) - self.assertIn("name: python-dist", section) - self.assertIn("path: ${{ needs.resolve.outputs.plugin_path }}/dist", section) - self.assertIn( - "working-directory: ${{ needs.resolve.outputs.plugin_path }}", section - ) - self.assertIn(f'"${{venv_python}}" -m pip install {artifact_glob}', section) - self.assertIn( - "needs: [resolve, build, test-built-wheel, test-built-sdist]", publish - ) - self.assertIn( - "needs.resolve.outputs.skip != 'true' && needs.resolve.outputs.publish_enabled == 'true'", - publish, - ) - self.assertIn( - "needs.resolve.outputs.publish_env != 'pypi' || needs.resolve.outputs.tag_on_main == 'true'", - publish, - ) + record = records[slug] + self.assertEqual(record.language, language) + self.assertEqual(record.path, path) + self.assertEqual(record.package_name, f"cpex-{slug.replace('_', '-')}") + self.assertEqual(record.module_name, f"cpex_{slug}") + self.assertEqual(record.cargo_package_name, slug if language == "rust" else None) + self.assertEqual(record.kind, f"cpex_{slug}.plugin.DemoPlugin") + self.assertEqual(record.version, "1.0.0") + + def test_requires_a_manifest(self) -> None: + (self.root / RUST_PATH / "cpex_rust_demo/plugin-manifest.yaml").unlink() + with self.assertRaisesRegex(catalog.CatalogError, "missing required path"): + catalog.discover_plugins(self.root) + + def test_rejects_duplicate_slugs_across_languages(self) -> None: + self.create_plugin("rust_demo", "python") + with self.assertRaisesRegex(catalog.CatalogError, "Duplicate plugin slug"): + catalog.discover_plugins(self.root) + + def test_ignores_manifests_outside_managed_roots(self) -> None: + outside = self.root / "examples/plugin-manifest.yaml" + outside.parent.mkdir() + outside.write_text("invalid example manifest\n") + self.assertEqual(len(catalog.discover_plugins(self.root)), 3) + + def test_rejects_invalid_plugin_slug(self) -> None: + plugin = self.root / RUST_PATH + plugin.rename(plugin.with_name("Invalid-Name")) + with self.assertRaisesRegex(catalog.CatalogError, "plugin slug"): + catalog.discover_plugins(self.root) + + def test_rejects_packaging_metadata_that_cannot_load_or_release(self) -> None: + cases = [ + (RUST_PATH, "pyproject.toml", 'name = "cpex-rust-demo"', 'name = "wrong"', "package name"), + (RUST_PATH, "Cargo.toml", 'name = "rust_demo"', 'name = "wrong"', r"\[package\].name"), + (RUST_PATH, "pyproject.toml", 'dynamic = ["version"]', 'version = "1.0.0"', "dynamic version"), + (RUST_PATH, "pyproject.toml", "rust_demo_rust", "wrong_rust", "module-name"), + (RUST_PATH, "pyproject.toml", 'python-source = "."', 'python-source = "src"', "python-source"), + (RUST_PATH, "Cargo.toml", "https://github.com/IBM/cpex-plugins", "https://example.test", "repository metadata"), + (PYTHON_PATH, "pyproject.toml", 'version = "1.0.0"', 'dynamic = ["version"]', "must be static"), + ] + for path, filename, old, new, error in cases: + with self.subTest(path=path, change=new): + file = self.root / path / filename + self.assert_invalid_edit(file, file.read_text().replace(old, new), error) + + def test_requires_matching_manifest_and_entry_point(self) -> None: + manifest = self.root / RUST_PATH / "cpex_rust_demo/plugin-manifest.yaml" + project = self.root / RUST_PATH / "pyproject.toml" + cases = [ + (manifest, 'version: "1.0.0"\n', "Missing kind"), + (manifest, 'version: "1.0.0"\nkind: "cpex_rust_demo.plugin.OtherPlugin"\n', "kind mismatch"), + (manifest, 'version: "1.0.0"\nkind: "cpex_rust_demo.plugin:DemoPlugin"\n', "canonical module.object"), + (project, project.read_text().replace('[project.entry-points."cpex.plugins"]', "[project.entry-points.other]"), "entry-points"), + (project, project.read_text().replace("plugin:DemoPlugin", "plugin.DemoPlugin"), "module:object"), + ] + for file, content, error in cases: + with self.subTest(error=error): + self.assert_invalid_edit(file, content, error) + + def test_requires_manifest_version_to_match_language_version_source(self) -> None: + for path, slug in ((RUST_PATH, "rust_demo"), (PYTHON_PATH, "python_demo")): + with self.subTest(slug=slug): + manifest = self.root / path / f"cpex_{slug}/plugin-manifest.yaml" + self.assert_invalid_edit(manifest, manifest.read_text().replace('"1.0.0"', '"2.0.0"'), "version mismatch") + + def test_accepts_manifest_comments(self) -> None: + manifest = self.root / RUST_PATH / "cpex_rust_demo/plugin-manifest.yaml" + manifest.write_text('version: "1.0.0" # release\nkind: cpex_rust_demo.plugin.DemoPlugin # loader\n') + catalog.discover_plugins(self.root) + + def test_enforces_language_appropriate_workspace_membership(self) -> None: + cases = [ + ("Cargo.toml", "[workspace]\nmembers = []\n", "missing from the top-level Cargo workspace"), + ("pyproject.toml", "[tool.uv.workspace]\nmembers = []\n", "missing from the root uv workspace"), + ("Cargo.toml", f'[workspace]\nmembers = ["{RUST_PATH}", "{OTHER_PATH}", "{PYTHON_PATH}"]\n', "must not be Cargo workspace members"), + ] + for filename, content, error in cases: + with self.subTest(filename=filename, error=error): + self.assert_invalid_edit(self.root / filename, content, error) - def test_all_plugins_share_the_root_uv_workspace_lock(self) -> None: - plugins = discover_plugins(REPO_ROOT) - root_project = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text()) - members = set(root_project["tool"]["uv"]["workspace"]["members"]) - self.assertEqual(members, {plugin.path for plugin in plugins}) - self.assertTrue((REPO_ROOT / "uv.lock").is_file()) - for plugin in plugins: - with self.subTest(plugin=plugin.slug): - self.assertFalse((REPO_ROOT / plugin.path / "uv.lock").exists()) + def test_malformed_metadata_reports_a_catalog_error(self) -> None: + cases = [ + ("Cargo.toml", "[workspace", "Invalid TOML"), + ("Cargo.toml", 'workspace = "invalid"\n', "must define"), + (f"{RUST_PATH}/pyproject.toml", "[project", "Invalid TOML"), + (f"{RUST_PATH}/pyproject.toml", 'project = "invalid"\n', "must be a table"), + (f"{RUST_PATH}/Cargo.toml", 'package = "invalid"\n', "must be a table"), + ] + for filename, content, error in cases: + with self.subTest(filename=filename, content=content): + self.assert_invalid_edit(self.root / filename, content, error) - def test_plugin_maintenance_runs_rust_and_python_plugin_lists(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "plugin-maintenance.yaml" - ).read_text() - run = self._extract_workflow_step_run( - workflow, - "update-and-test", - step_name="Test all plugins", - ) - arrays = { - name: re.search( - rf"(?m)^{name}=\(\n(?P(?: [a-z0-9_]+\n)+)\)$", - run, - ) - for name in ("rust_plugins", "python_plugins") - } - for name, match in arrays.items(): - self.assertIsNotNone(match, f"expected shell array {name!r}") - rust_plugins = re.findall(r"(?m)^ ([a-z0-9_]+)$", arrays["rust_plugins"].group("body")) - python_plugins = re.findall( - r"(?m)^ ([a-z0-9_]+)$", - arrays["python_plugins"].group("body"), - ) +class SelectionTests(CatalogTestCase): + def test_all_mode_reports_complete_language_and_job_contract(self) -> None: self.assertEqual( - rust_plugins, - [ - "encoded_exfil_detection", - "output_length_guard", - "pii_filter", - "rate_limiter", - "retry_with_backoff", - "secrets_detection", - "sql_sanitizer", - "url_reputation", - ], - ) - self.assertEqual(python_plugins, ["ica_metering_exporter"]) - - rust_loop = re.search( - r'(?ms)^for plugin in "\$\{rust_plugins\[@\]\}"; do\n(?P.+?)^done$', - run, - ) - python_loop = re.search( - r'(?ms)^for plugin in "\$\{python_plugins\[@\]\}"; do\n(?P.+?)^done$', - run, - ) - self.assertIsNotNone(rust_loop, "expected rust_plugins loop") - self.assertIsNotNone(python_loop, "expected python_plugins loop") - self.assertIn('pushd "plugins/rust/python-package/${plugin}"', rust_loop.group("body")) - self.assertNotIn('pushd "plugins/python/${plugin}"', rust_loop.group("body")) - self.assertIn('pushd "plugins/python/${plugin}"', python_loop.group("body")) - self.assertNotIn( - 'pushd "plugins/rust/python-package/${plugin}"', - python_loop.group("body"), - ) - - def test_catalog_workflow_runs_catalog_suite(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-plugin-catalog.yaml" - ).read_text() - self.assertIn("name: CI Plugin Catalog", workflow) - self.assertIn("concurrency:", workflow) - self.assertIn("cancel-in-progress: true", workflow) - self.assertIn("python3 -m unittest tests/test_plugin_catalog.py", workflow) - self.assertNotIn("tests/test_install_built_wheel.py", workflow) - self.assertNotIn("python3 tools/plugin_catalog.py ci-selection", workflow) - self.assertIn( - "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1", - workflow, - ) - self.assertIn( - "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0", - workflow, - ) - - def test_install_wheel_workflow_runs_install_suite(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-install-built-wheel.yaml" - ).read_text() - self.assertIn("name: CI Install Built Wheel", workflow) - self.assertIn("concurrency:", workflow) - self.assertIn("cancel-in-progress: true", workflow) - self.assertIn("python3 -m unittest tests/test_install_built_wheel.py", workflow) - self.assertNotIn("tests/test_plugin_catalog.py", workflow) - self.assertIn( - "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1", - workflow, - ) - self.assertIn( - "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0", - workflow, - ) - - def test_ci_workflow_includes_parity_jobs_for_rust_plugin_checks(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-rust-python-package.yaml" - ).read_text() - security_section = self._extract_workflow_job_section(workflow, "security-policy") - mutants_section = self._extract_workflow_job_section(workflow, "mutation-testing") - coverage_section = self._extract_workflow_job_section(workflow, "coverage") - documentation_section = self._extract_workflow_job_section( - workflow, "documentation" - ) - create_tags_section = self._extract_workflow_job_section( - workflow, "create-release-tags" - ) - release_validation_section = self._extract_workflow_job_section( - workflow, "release-validation" - ) - detect_run = self._extract_workflow_step_run( - workflow, "validate-and-detect", step_id="detect" - ) - deny_run = self._extract_workflow_step_run( - workflow, "security-policy", step_name="Run cargo deny" - ) - mutants_install_run = self._extract_workflow_step_run( - workflow, "mutation-testing", step_name="Install Rust mutation testing tooling" - ) - mutants_diff_run = self._extract_workflow_step_run( - workflow, "mutation-testing", step_name="Create mutation diff" - ) - mutants_run = self._extract_workflow_step_run( - workflow, "mutation-testing", step_name="Run cargo-mutants with nextest" - ) - build_test_nextest_install = self._extract_workflow_step_section( - workflow, "build-test", step_name="Install cargo-nextest" - ) - build_test_ci_build = self._extract_workflow_step_section( - workflow, "build-test", step_name="Plugin CI build verification" - ) - build_test_ci = self._extract_workflow_step_section( - workflow, "build-test", step_name="Plugin CI verification" - ) - coverage_run = self._extract_workflow_step_run( - workflow, "coverage", step_name="Generate Rust coverage report" - ) - coverage_check_run = self._extract_workflow_step_run( - workflow, "coverage", step_name="Enforce per-plugin coverage floor" - ) - documentation_run = self._extract_workflow_step_run( - workflow, "documentation", step_name="Build Rust documentation" - ) - - self.assertIn("workflow_dispatch:", workflow) - self.assertIn('elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then', detect_run) - self.assertIn("ci-selection . all '' ''", detect_run) - self.assertIn( - "ci-selection . diff \"${{ github.event.pull_request.base.sha }}\" \"${{ github.event.pull_request.head.sha }}\"", - detect_run, - ) - self.assertIn("rust_plugin_count: ${{ steps.detect.outputs.rust_plugin_count }}", workflow) - self.assertNotIn("single_cargo_package", workflow) - self.assertIn("cargo_packages: ${{ steps.detect.outputs.cargo_packages }}", workflow) - self.assertIn("mutation_cargo_packages: ${{ steps.detect.outputs.mutation_cargo_packages }}", workflow) - self.assertIn("mutation_jobs: ${{ steps.detect.outputs.mutation_jobs }}", workflow) - self.assertIn("has_mutation_cargo_packages: ${{ steps.detect.outputs.has_mutation_cargo_packages }}", workflow) - self.assertIn( - "rust_release_validation_tags: ${{ steps.detect.outputs.rust_release_validation_tags }}", - workflow, - ) - self.assertIn( - "has_rust_release_validation_tags: ${{ steps.detect.outputs.has_rust_release_validation_tags }}", - workflow, - ) - self.assertIn("security-policy:", workflow) - self.assertIn("mutation-testing:", workflow) - self.assertIn("coverage:", workflow) - self.assertIn("documentation:", workflow) - self.assertNotIn("benchmark-build-verification:", workflow) - self.assertIn("if: needs.validate-and-detect.outputs.has_plugins == 'true'", security_section) - self.assertIn("if: github.event_name == 'pull_request' && needs.validate-and-detect.outputs.has_mutation_cargo_packages == 'true'", mutants_section) - self.assertIn("if: needs.validate-and-detect.outputs.has_rust_plugins == 'true'", coverage_section) - self.assertIn("if: needs.validate-and-detect.outputs.has_plugins == 'true'", documentation_section) - self.assertIn("if: github.event_name == 'pull_request' && needs.validate-and-detect.outputs.has_rust_release_validation_tags == 'true'", release_validation_section) - self.assertIn("tag: ${{ fromJson(needs.validate-and-detect.outputs.rust_release_validation_tags) }}", release_validation_section) - self.assertIn("github.event_name == 'push'", create_tags_section) - self.assertIn("github.ref == 'refs/heads/main'", create_tags_section) - self.assertIn("always()", create_tags_section) - self.assertIn("needs.build-test.result == 'success'", create_tags_section) - self.assertIn("needs.security-policy.result == 'success'", create_tags_section) - self.assertIn("needs.coverage.result == 'success'", create_tags_section) - self.assertIn("needs.documentation.result == 'success'", create_tags_section) - self.assertIn("needs.mutation-testing.result == 'skipped'", create_tags_section) - self.assertIn("needs.release-validation.result == 'skipped'", create_tags_section) - self.assertIn("actions: write", create_tags_section) - self.assertIn("contents: write", create_tags_section) - # Keep the tag loop out of a pipeline so bash exits on failures inside the loop. - self.assertNotIn("mapfile", create_tags_section) - self.assertIn("done < <(python3 -c", create_tags_section) - self.assertIn('[[ ! "${tag}" =~ ^[a-zA-Z0-9._-]+$ ]]', create_tags_section) - self.assertIn('git tag "${tag}" "${GITHUB_SHA}"', create_tags_section) - self.assertIn('git push origin "refs/tags/${tag}"', create_tags_section) - self.assertIn("GH_TOKEN: ${{ github.token }}", create_tags_section) - self.assertIn("gh workflow run release-rust-python-package.yaml", create_tags_section) - self.assertIn('--ref "${tag}"', create_tags_section) - self.assertNotIn("--ref main", create_tags_section) - self.assertIn('-f "tag=${tag}"', create_tags_section) - self.assertIn("-f repository=pypi", create_tags_section) - self.assertIn("-f publish_enabled=true", create_tags_section) - self.assertIn("gh run list -w release-rust-python-package.yaml", create_tags_section) - self.assertIn("--json url --jq '.[0].url'", create_tags_section) - self.assertLess( - create_tags_section.index('git push origin "refs/tags/${tag}"'), - create_tags_section.index("gh workflow run release-rust-python-package.yaml"), - ) - self.assertNotIn("publish-release-tags:", workflow) - self.assertNotIn("cargo-audit", security_section) - self.assertNotIn("cargo audit", security_section) - self.assertIn("cargo install cargo-deny", security_section) - self.assertEqual("cargo deny --all-features check --config deny.toml\n", deny_run) - self.assertNotIn("-A unmaintained", security_section) - self.assertNotIn("--workspace", security_section) - self.assertNotIn("--manifest-path", security_section) - self.assertNotIn("matrix:", security_section) - self.assertIn("NEXTEST_PROFILE: ci", build_test_ci_build) - self.assertIn("NEXTEST_PROFILE: ci", build_test_ci) - self.assertIn("cargo install cargo-llvm-cov --version 0.8.4 --locked", coverage_section) - self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", build_test_nextest_install) - self.assertIn("cargo nextest --version", build_test_nextest_install) - self.assertIn("https://get.nexte.st/0.9.133/mac", build_test_nextest_install) - self.assertNotIn("mac-arm", workflow) - self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", coverage_section) - self.assertIn("cargo nextest --version", coverage_section) - self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", mutants_install_run) - self.assertIn("cargo nextest --version", mutants_install_run) - self.assertIn("cargo install cargo-mutants --version 27.0.0 --locked", mutants_install_run) - self.assertIn("cargo mutants --version", mutants_install_run) - self.assertIn("fetch-depth: 0", mutants_section) - self.assertEqual('git diff "${BASE_SHA}" -- \'*.rs\' > cargo-mutants.diff\n', mutants_diff_run) - self.assertIn("BASE_SHA: ${{ github.event.pull_request.base.sha }}", mutants_section) - self.assertNotIn("HEAD_SHA:", mutants_section) - self.assertIn("mutation_job: ${{ fromJson(needs.validate-and-detect.outputs.mutation_jobs) }}", mutants_section) - self.assertIn("CARGO_PACKAGE: ${{ matrix.mutation_job.cargo_package }}", mutants_section) - self.assertIn("IN_DIFF: ${{ matrix.mutation_job.in_diff }}", mutants_section) - self.assertIn("TEST_PACKAGES: ${{ toJson(matrix.mutation_job.test_packages) }}", mutants_section) - self.assertIn('cargo_args=("-p" "${CARGO_PACKAGE}")', mutants_run) - self.assertIn('cargo_args+=("--in-diff" "cargo-mutants.diff")', mutants_run) - self.assertIn('cargo_args+=("--test-package" "${package}")', mutants_run) - self.assertIn('cargo mutants "${cargo_args[@]}"', mutants_run) - self.assertIn("PYO3_PYTHON: python", mutants_section) - self.assertIn("python -m pip install uv==0.9.30 maturin==1.12.6", coverage_section) - self.assertIn("CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }}", coverage_section) - self.assertIn("PLUGINS: ${{ needs.validate-and-detect.outputs.rust_plugins }}", coverage_section) - self.assertIn('os.environ["CARGO_PACKAGES"]', coverage_run) - self.assertIn('os.environ["PLUGINS"]', coverage_run) - self.assertIn('cargo_args+=("-p" "${package}")', coverage_run) - self.assertIn("cargo llvm-cov clean --workspace", coverage_run) - self.assertIn('eval "$(cargo llvm-cov show-env --sh)"', coverage_run) - self.assertIn( - 'CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target"', - coverage_run, - ) - self.assertIn( - 'CARGO_LLVM_COV_BUILD_DIR="${CARGO_TARGET_DIR}"', - coverage_run, - ) - self.assertIn( - 'LLVM_PROFILE_FILE="${CARGO_TARGET_DIR}/cpex-plugins-%p-%10m.profraw"', - coverage_run, - ) - self.assertIn( - 'mkdir -p "${CARGO_TARGET_DIR}"', - coverage_run, - ) - self.assertIn( - 'cargo llvm-cov nextest --no-report "${cargo_args[@]}" -P "${NEXTEST_PROFILE}"', - coverage_run, - ) - self.assertIn('make sync && uv run maturin develop', coverage_run) - self.assertIn('make test-integration', coverage_run) - self.assertIn( - 'env -u CARGO_TARGET_DIR -u CARGO_LLVM_COV_BUILD_DIR -u CARGO_LLVM_COV_TARGET_DIR -u LLVM_PROFILE_FILE cargo llvm-cov report "${cargo_args[@]}" --cobertura --output-path coverage/cobertura.xml', - coverage_run, - ) - self.assertNotIn("cargo llvm-cov --workspace", coverage_run) - self.assertIn("python3 tools/plugin_catalog.py coverage-check . coverage/cobertura.xml 90.00", coverage_check_run) - self.assertIn('"${PLUGINS}"', coverage_check_run) - self.assertIn("cobertura.xml", coverage_section) - self.assertNotIn("matrix:", coverage_section) - self.assertIn("CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }}", documentation_section) - self.assertIn('os.environ["CARGO_PACKAGES"]', documentation_run) - self.assertIn('cargo_args+=("-p" "${package}")', documentation_run) - self.assertIn('cargo doc "${cargo_args[@]}" --lib --no-deps --document-private-items', documentation_run) - self.assertNotIn("cargo doc --workspace", documentation_run) - self.assertNotIn("matrix:", documentation_section) - self.assertNotIn("upload-artifact", documentation_section) - - def test_ci_workflow_dispatch_detect_step_selects_all_plugins(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-rust-python-package.yaml" - ).read_text() - detect_run = self._extract_workflow_step_run( - workflow, "validate-and-detect", step_id="detect" - ) - with tempfile.TemporaryDirectory() as tmpdir: - output_path = Path(tmpdir) / "github_output" - result = subprocess.run( - ["bash", "-c", detect_run], - cwd=REPO_ROOT, - env={ - **os.environ, - "GITHUB_EVENT_NAME": "workflow_dispatch", - "GITHUB_OUTPUT": str(output_path), - }, - text=True, - capture_output=True, - check=False, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - outputs = dict( - line.split("=", maxsplit=1) - for line in output_path.read_text().splitlines() - if "=" in line - ) - self.assertEqual(outputs["has_plugins"], "true") - self.assertEqual(outputs["plugin_count"], "9") - expected_rust_plugins = [ - "encoded_exfil_detection", - "output_length_guard", - "pii_filter", - "rate_limiter", - "retry_with_backoff", - "secrets_detection", - "sql_sanitizer", - "url_reputation", - ] - expected_plugins = [ - "encoded_exfil_detection", - "ica_metering_exporter", - *expected_rust_plugins[1:], - ] - self.assertEqual(json.loads(outputs["plugins"]), expected_plugins) - self.assertEqual(json.loads(outputs["cargo_packages"]), expected_rust_plugins) - self.assertEqual( - json.loads(outputs["mutation_cargo_packages"]), expected_rust_plugins - ) - self.assertEqual( - json.loads(outputs["mutation_jobs"]), - [ - {"cargo_package": plugin, "in_diff": False, "test_packages": []} - for plugin in expected_rust_plugins + catalog.ci_selection(self.root, "all"), + { + "plugins": ALL_PLUGINS, + "rust_plugins": ["other_demo", "rust_demo"], + "python_plugins": ["python_demo"], + "has_plugins": True, + "has_rust_plugins": True, + "has_python_plugins": True, + "plugin_count": 3, + "rust_plugin_count": 2, + "python_plugin_count": 1, + "cargo_packages": ["other_demo", "rust_demo"], + "mutation_cargo_packages": ["other_demo", "rust_demo"], + "has_mutation_cargo_packages": True, + "mutation_jobs": [ + {"cargo_package": "other_demo", "in_diff": False, "test_packages": []}, + {"cargo_package": "rust_demo", "in_diff": False, "test_packages": []}, ], - ) - self.assertEqual(outputs["has_mutation_cargo_packages"], "true") - - def test_scaffold_workflow_installs_nextest_for_generated_plugin_ci(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "ci-scaffold-generator.yaml" - ).read_text() - nextest_install = self._extract_workflow_step_section( - workflow, "test", step_name="Install cargo-nextest" + "release_validation_tags": [], + "rust_release_validation_tags": [], + "python_release_validation_tags": [], + "has_release_validation_tags": False, + "has_rust_release_validation_tags": False, + "has_python_release_validation_tags": False, + }, ) - self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", nextest_install) - self.assertIn("https://get.nexte.st/0.9.133/mac", nextest_install) - self.assertNotIn("mac-arm", workflow) - self.assertIn("cargo nextest --version", nextest_install) - - def test_testing_docs_include_local_rust_coverage_command(self) -> None: - testing_doc = (REPO_ROOT / "TESTING.md").read_text() - mutants_config = (REPO_ROOT / ".cargo" / "mutants.toml").read_text() - nextest_config = (REPO_ROOT / ".config" / "nextest.toml").read_text() - self.assertIn("cargo install cargo-llvm-cov --version 0.8.4 --locked", testing_doc) - self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", testing_doc) - self.assertIn("cargo llvm-cov clean --workspace", testing_doc) - self.assertIn('eval "$(cargo llvm-cov show-env --sh)"', testing_doc) - self.assertIn('export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target"', testing_doc) - self.assertIn("make sync && uv run maturin develop", testing_doc) - self.assertIn("cargo llvm-cov nextest --no-report", testing_doc) - self.assertIn("make test-integration", testing_doc) - self.assertIn("env -u CARGO_TARGET_DIR", testing_doc) - self.assertIn("coverage/cobertura.xml", testing_doc) - self.assertIn("python3 tools/plugin_catalog.py coverage-check . coverage/cobertura.xml 90.00", testing_doc) - self.assertIn("Nextest does not run Rust doctests", testing_doc) - self.assertIn("cargo nextest run --benches -E 'kind(bench)' --no-run", testing_doc) - self.assertIn("cargo install cargo-nextest --version 0.9.133 --locked", testing_doc) - self.assertIn("cargo install cargo-mutants --version 27.0.0 --locked", testing_doc) - self.assertIn("make plugin-mutants PLUGIN=retry_with_backoff", testing_doc) - self.assertIn('cargo mutants "${cargo_args[@]}"', testing_doc) - self.assertIn('test_tool = "nextest"', mutants_config) - self.assertIn('additional_cargo_test_args = ["--profile=mutants"]', mutants_config) - self.assertIn("cap_lints = false", mutants_config) - self.assertIn('nextest-version = "0.9.133"', nextest_config) - self.assertIn("[profile.ci]", nextest_config) - self.assertIn("fail-fast = false", nextest_config) - self.assertIn('failure-output = "immediate-final"', nextest_config) - self.assertIn("[profile.mutants]", nextest_config) - self.assertIn("fail-fast = true", nextest_config) - - def test_python_integration_tests_live_under_repo_plugins_tests(self) -> None: - plugin_root = REPO_ROOT / "plugins" / "rust" / "python-package" - integration_root = REPO_ROOT / "plugins" / "tests" - for slug in ( - "encoded_exfil_detection", - "pii_filter", - "rate_limiter", - "retry_with_backoff", - "secrets_detection", - "url_reputation", + def test_diff_selects_affected_plugins(self) -> None: + cases = [ + ([f"{RUST_PATH}/src/lib.rs"], ["rust_demo"]), + ([f"{PYTHON_PATH}/cpex_python_demo/plugin.py"], ["python_demo"]), + ([f"{RUST_PATH}/README.md", f"{PYTHON_PATH}/README.md"], ["python_demo", "rust_demo"]), + (["plugins/tests/rust_demo/test_plugin.py"], ["rust_demo"]), + (["plugins/tests/python_demo/test_plugin.py"], ["python_demo"]), + (["plugins/tests/conftest.py"], ALL_PLUGINS), + (["plugins/tests/unknown/test_plugin.py"], ALL_PLUGINS), + (["Cargo.lock"], ALL_PLUGINS), + (["Cargo.lock", f"{RUST_PATH}/src/lib.rs"], ["rust_demo"]), + (["Makefile"], ALL_PLUGINS), + (["Cargo.toml"], ALL_PLUGINS), + (["pyproject.toml"], ALL_PLUGINS), + (["uv.lock"], ALL_PLUGINS), + ([".github/workflows/ci-rust-python-package.yaml"], ALL_PLUGINS), + ([".cargo/mutants.toml"], ALL_PLUGINS), + ([".config/nextest.toml"], ALL_PLUGINS), + (["deny.toml"], ALL_PLUGINS), + (["tools/plugin_catalog.py"], ALL_PLUGINS), + (["TESTING.md"], ALL_PLUGINS), + (["tests/test_plugin_catalog.py"], []), + (["CONTRIBUTING.md"], []), + ([], []), + ] + for paths, expected in cases: + with self.subTest(paths=paths): + result = self.selection(paths) + self.assertEqual(result["plugins"], expected) + self.assertEqual(result["has_plugins"], bool(expected)) + self.assertEqual(result["plugin_count"], len(expected)) + + def test_mutation_jobs_follow_rust_source_changes(self) -> None: + for paths, expected in ( + ([f"{RUST_PATH}/src/lib.rs"], [{"cargo_package": "rust_demo", "in_diff": True, "test_packages": []}]), + ([f"{PYTHON_PATH}/cpex_python_demo/plugin.py"], []), + (["plugins/tests/rust_demo/test_plugin.py"], []), + ([".config/nextest.toml"], []), ): - self.assertTrue((integration_root / slug).exists(), slug) - self.assertFalse((plugin_root / slug / "tests").exists(), slug) - - makefile = (plugin_root / slug / "Makefile").read_text() - pyproject = (plugin_root / slug / "pyproject.toml").read_text() - self.assertIn(f"../../../tests/{slug}", makefile) - self.assertIn("CPEX_TEST_PLUGIN_HOOKS=1", makefile) - self.assertIn("test-integration", makefile) - self.assertRegex(makefile, r"(?m)^test-unit:\n\t@echo") - self.assertIn("test: test-unit test-integration", makefile) - self.assertIn("test-all: test", makefile) - self.assertIn("check-all: fmt-check clippy test-unit", makefile) - self.assertIn(f"CARGO_PACKAGE := {slug}", makefile) - self.assertIn("NEXTEST_PROFILE ?= default", makefile) - self.assertNotIn("[tool.pytest.ini_options]", pyproject) - - result = subprocess.run( - ["make", "-n", "test"], - cwd=plugin_root / slug, - text=True, - capture_output=True, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn(f"cargo nextest run --profile default -p {slug}", result.stdout) - self.assertIn(f"../../../tests/{slug}", result.stdout) - - def test_coverage_check_reports_per_plugin_percentages(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/alpha", "plugins/rust/python-package/beta"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "alpha") - self._create_plugin(root, "beta") - report = root / "coverage.xml" - report.write_text( - textwrap.dedent( - """ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - """ - ).strip() - ) - - result = run_catalog( - "coverage-check", str(root), str(report), "50.0", '["alpha", "beta"]' - ) - - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["minimum_plugin"], "alpha") - self.assertEqual(payload["minimum_line_rate"], 50.0) - self.assertEqual(payload["plugins"]["alpha"]["line_rate"], 50.0) - self.assertEqual(payload["plugins"]["beta"]["line_rate"], 75.0) - - def test_coverage_check_accepts_windows_paths(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/alpha"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "alpha") - report = root / "coverage.xml" - report.write_text( - textwrap.dedent( - r""" - - - - - - - - - - - - - - - """ - ).strip() - ) - - result = run_catalog("coverage-check", str(root), str(report), "50.0", '["alpha"]') - - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["plugins"]["alpha"]["line_rate"], 50.0) - - def test_coverage_check_rejects_plugin_with_no_counted_lines(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/alpha"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "alpha") - report = root / "coverage.xml" - report.write_text( - textwrap.dedent( - """ - - - - - - - - - - - - """ - ).strip() - ) - - result = run_catalog("coverage-check", str(root), str(report), "50.0", '["alpha"]') - - self.assertEqual(result.returncode, 1) - self.assertIn("alpha has no counted coverage lines", result.stderr) - - def test_coverage_check_fails_below_threshold(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/alpha"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "alpha") - report = root / "coverage.xml" - report.write_text( - textwrap.dedent( - """ - - - - - - - - - - - - - - - """ - ).strip() - ) - - result = run_catalog("coverage-check", str(root), str(report), "50.1", '["alpha"]') - - self.assertEqual(result.returncode, 1) - self.assertIn("alpha line coverage 50.00% is below 50.10%", result.stderr) - - def test_coverage_check_rejects_unmanaged_plugin_coverage(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/alpha"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "alpha") - report = root / "coverage.xml" - report.write_text( - textwrap.dedent( - """ - - - - - - - - - - - - """ - ).strip() - ) - - result = run_catalog("coverage-check", str(root), str(report), "50.0", '["alpha"]') - - self.assertEqual(result.returncode, 1) - self.assertIn("unknown plugin coverage entries: ['ghost']", result.stderr) - - def test_coverage_check_requires_expected_plugins(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/alpha", "plugins/rust/python-package/beta"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "alpha") - self._create_plugin(root, "beta") - report = root / "coverage.xml" - report.write_text( - textwrap.dedent( - """ - - - - - - - - - - - - """ - ).strip() - ) - - result = run_catalog( - "coverage-check", str(root), str(report), "50.0", '["alpha", "beta"]' - ) - - self.assertEqual(result.returncode, 1) - self.assertIn("missing plugin coverage entries: ['beta']", result.stderr) - - def test_coverage_check_rejects_unexpected_managed_plugin_coverage(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/alpha", "plugins/rust/python-package/beta"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "alpha") - self._create_plugin(root, "beta") - report = root / "coverage.xml" - report.write_text( - textwrap.dedent( - """ - - - - - - - - - - - - - - - - - - - """ - ).strip() - ) - - result = run_catalog("coverage-check", str(root), str(report), "50.0", '["alpha"]') - - self.assertEqual(result.returncode, 1) - self.assertIn("unexpected plugin coverage entries: ['beta']", result.stderr) - - def test_coverage_check_reports_malformed_hit_counts_without_traceback(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/rust/python-package/alpha"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "alpha") - report = root / "coverage.xml" - report.write_text( - textwrap.dedent( - """ - - - - - - - - - - - - """ - ).strip() - ) - - result = run_catalog("coverage-check", str(root), str(report), "50.0", '["alpha"]') - - self.assertEqual(result.returncode, 1) - self.assertIn("Invalid coverage hit count 'NaN'", result.stderr) - self.assertNotIn("Traceback", result.stderr) - - def test_workspace_has_single_cargo_deny_config(self) -> None: - root_deny = REPO_ROOT / "deny.toml" - self.assertTrue(root_deny.exists()) - for plugin_dir in (REPO_ROOT / "plugins" / "rust" / "python-package").iterdir(): - if plugin_dir.is_dir(): - self.assertFalse( - (plugin_dir / "deny.toml").exists(), - f"expected workspace-level deny.toml only, found {plugin_dir / 'deny.toml'}", - ) - - def test_release_workflow_tests_artifacts_outside_source_tree(self) -> None: - workflow = ( - REPO_ROOT / ".github" / "workflows" / "release-rust-python-package.yaml" - ).read_text() - preflight_section = self._extract_workflow_job_section(workflow, "preflight") - build_wheel_section = self._extract_workflow_job_section(workflow, "build-wheel") - self.assertIn("preflight:", workflow) - self.assertIn("needs: [resolve, preflight]", workflow) - self.assertIn("shell: bash", workflow) - self.assertIn("rustc --version", workflow) - self.assertIn("working-directory: ${{ needs.resolve.outputs.plugin_path }}", workflow) - self.assertIn( - 'if [[ -d "${GITHUB_WORKSPACE}/plugins/tests/${{ needs.resolve.outputs.slug }}" ]]; then', - workflow, - ) - self.assertIn( - 'cp -R "${GITHUB_WORKSPACE}/plugins/tests/${{ needs.resolve.outputs.slug }}" "${tmpdir}/tests/${{ needs.resolve.outputs.slug }}"', - workflow, - ) - self.assertIn('cp "${GITHUB_WORKSPACE}/plugins/tests/conftest.py"', workflow) - self.assertIn('cp "${GITHUB_WORKSPACE}/plugins/tests/plugin_hooks.py"', workflow) - self.assertIn('cp "${GITHUB_WORKSPACE}/plugins/tests/pytest.ini"', workflow) - self.assertIn("CPEX_TEST_PLUGIN_HOOKS=1", workflow) - self.assertIn('PYTHONPATH="${tmpdir}/tests"', workflow) - self.assertIn('cd "${tmpdir}"', workflow) - self.assertIn('"${tmpdir}/tests/${{ needs.resolve.outputs.slug }}" -v', workflow) - self.assertNotIn('PYTHONPATH="${GITHUB_WORKSPACE}/${{ needs.resolve.outputs.plugin_path }}/tests"', workflow) - self.assertNotIn("cargo run --features stub-gen --bin stub_gen", workflow) - self.assertIn('test -n "$(find . -name \'*.pyi\' -print -quit)"', workflow) - self.assertIn('git ls-remote --exit-code --tags origin "refs/tags/${tag}"', workflow) - self.assertIn('elif [[ "${GITHUB_EVENT_NAME}" == "pull_request" && "${PUBLISH_ENABLED}" == "false" ]]; then', workflow) - self.assertIn('checkout_ref="${GITHUB_SHA}"', workflow) - self.assertIn('echo "Release tag ${tag} does not exist" >&2', workflow) - self.assertIn("python3 tools/plugin_catalog.py release-info .", workflow) - self.assertIn('if [[ -n "${TAG_INPUT}" ]]; then', workflow) - self.assertIn("workflow_call:", workflow) - self.assertIn("publish_enabled:", workflow) - self.assertIn('default: false', workflow) - self.assertIn("workflow_dispatch:", workflow) - self.assertIn('default: true', workflow) - self.assertIn('git fetch --force origin "refs/heads/main:refs/remotes/origin/main"', workflow) - self.assertIn('if git merge-base --is-ancestor "${tag_ref}" "refs/remotes/origin/main"; then', workflow) - self.assertIn("tag_on_main: ${{ steps.resolve.outputs.tag_on_main }}", workflow) - self.assertIn("slug: ${{ steps.resolve.outputs.plugin }}", workflow) - self.assertIn("publish_enabled: ${{ steps.resolve.outputs.publish_enabled }}", workflow) - self.assertIn('echo "publish_enabled=false"', workflow) - self.assertIn('echo "publish_enabled=true"', workflow) - self.assertIn( - 'wheel_matrix="$(python3 -c \'import json; print(json.dumps([{', - workflow, - ) - self.assertIn( - '{"runner":"ubuntu-24.04-s390x","platform":"linux-s390x"}', - workflow, - ) - self.assertIn( - '{"runner":"ubuntu-24.04-ppc64le","platform":"linux-ppc64le"}', - workflow, - ) - self.assertIn( - 'wheel_matrix="$(printf \'%s\' "${release_info}" | python3 -c \'import json, sys; print(json.dumps(json.load(sys.stdin)["release_wheel_matrix"]))\')"', - workflow, - ) - self.assertIn("wheel_matrix: ${{ steps.resolve.outputs.wheel_matrix }}", workflow) - self.assertIn("matrix:\n include: ${{ fromJson(needs.resolve.outputs.wheel_matrix) }}", workflow) - self.assertIn("runs-on: ${{ matrix.runner }}", workflow) - self.assertIn("name: wheel-${{ matrix.platform }}", workflow) - self.assertNotIn("allow_non_main_pypi", workflow) - self.assertNotIn("ALLOW_NON_MAIN_PYPI", workflow) - self.assertIn( - "if: ${{ needs.resolve.outputs.skip != 'true' && needs.resolve.outputs.publish_enabled == 'true' && (needs.resolve.outputs.publish_env != 'pypi' || needs.resolve.outputs.tag_on_main == 'true') }}", - workflow, - ) - self.assertNotIn("matrix.", preflight_section) - self.assertIn( - "matrix.runner != 'ubuntu-24.04-s390x' && matrix.runner != 'ubuntu-24.04-ppc64le'", - build_wheel_section, - ) - self.assertIn( - "matrix.runner == 'ubuntu-24.04-s390x' || matrix.runner == 'ubuntu-24.04-ppc64le'", - build_wheel_section, - ) - self.assertIn( - "sudo apt-get install -y python3.12 python3.12-dev python3.12-venv python3-pip", - build_wheel_section, - ) - self.assertIn( - 'ln -sf "$(which python3.12)" "${python_bin_dir}/python"', - build_wheel_section, - ) - self.assertIn("sudo apt-get clean", build_wheel_section) - self.assertIn("sudo rm -rf /var/lib/apt/lists/*", build_wheel_section) - self.assertIn('export PATH="${python_bin_dir}:$PATH"', build_wheel_section) - self.assertNotIn('python -m ensurepip --upgrade', build_wheel_section) - self.assertIn('python -m pip --version', build_wheel_section) - self.assertNotIn("python -m pip install --upgrade pip", build_wheel_section) - self.assertNotIn("tools/plugin_catalog.py release-info-field", workflow) - self.assertNotIn("python3 - <<'PY'", workflow) - self.assertIn("uv==0.9.30", workflow) - self.assertIn("maturin==1.12.6", workflow) - self.assertNotIn("dtolnay/rust-toolchain", workflow) - self.assertIn( - 'venv_python="${tmpdir}/venv/Scripts/python.exe"', - workflow, - ) - self.assertIn('uv pip install --python "${venv_python}" --group dev PyYAML', workflow) - self.assertIn('"${venv_python}" -m pip install', workflow) - self.assertNotIn('pytest pytest-asyncio PyYAML', workflow) - self.assertIn('"${venv_python}" -m pytest', workflow) - self.assertIn( - "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1", - workflow, - ) - self.assertIn( - "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0", - workflow, - ) - self.assertIn( - "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1", - workflow, - ) - self.assertIn( - "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1", - workflow, - ) - self.assertLess( - workflow.index("name: Upload wheel artifact"), - workflow.index("name: Test built wheel in isolated virtualenv"), - ) - self.assertLess( - workflow.index("name: Upload sdist artifact"), - workflow.index("name: Test built sdist in isolated virtualenv"), - ) - self.assertIn( - "pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2", - workflow, - ) - self.assertNotIn("actions/checkout@v4", workflow) - self.assertNotIn("actions/setup-python@v5", workflow) - self.assertNotIn("actions/upload-artifact@v4", workflow) - self.assertNotIn("actions/download-artifact@v4", workflow) - self.assertNotIn("pypa/gh-action-pypi-publish@release/v1", workflow) - self.assertNotRegex( - workflow, - r"defaults:\n\s+run:\n\s+shell: bash\n\s+working-directory: .*\$\{\{", - ) - - def test_top_level_stub_exports_match_runtime_packages(self) -> None: - pii_stub = ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "pii_filter" - / "cpex_pii_filter" - / "__init__.pyi" - ).read_text() - rate_stub = ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "rate_limiter" - / "cpex_rate_limiter" - / "__init__.pyi" - ).read_text() - self.assertIn("PIIFilterPlugin", pii_stub) - self.assertIn("PIIDetectorRust", pii_stub) - self.assertIn("RateLimiterConfig", rate_stub) - self.assertIn("RateLimiterPlugin", rate_stub) - self.assertIn("_parse_rate", rate_stub) - self.assertNotIn("PIIFilterPluginCore", pii_stub) - self.assertNotIn("RateLimiterPluginCore", rate_stub) - - def test_extension_stubs_match_runtime_extension_exports(self) -> None: - pii_ext_stub = ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "pii_filter" - / "cpex_pii_filter" - / "pii_filter_rust" - / "__init__.pyi" - ).read_text() - rate_ext_stub = ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "rate_limiter" - / "cpex_rate_limiter" - / "rate_limiter_rust" - / "__init__.pyi" - ).read_text() - self.assertIn("PIIFilterPluginCore", pii_ext_stub) - self.assertIn("RateLimiterPluginCore", rate_ext_stub) - self.assertIn("compat_default_config", rate_ext_stub) - self.assertIn("compat_parse_rate", rate_ext_stub) - - def test_pii_filter_has_single_authoritative_extension_stub(self) -> None: - plugin_dir = ( - REPO_ROOT / "plugins" / "rust" / "python-package" / "pii_filter" - ) - self.assertTrue( - ( - plugin_dir - / "cpex_pii_filter" - / "pii_filter_rust" - / "__init__.pyi" - ).exists() - ) - self.assertFalse( - (plugin_dir / "python" / "pii_filter_rust" / "__init__.pyi").exists() - ) - - def test_pii_package_imports_without_mcpgateway(self) -> None: - plugin_dir = ( - REPO_ROOT / "plugins" / "rust" / "python-package" / "pii_filter" - ) - result = subprocess.run( - [ - "python3", - "-c", - "import cpex_pii_filter; print(cpex_pii_filter.__name__)", - ], - cwd=plugin_dir, - text=True, - capture_output=True, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "cpex_pii_filter") - - def test_pii_detector_export_is_available_when_extension_is_built(self) -> None: - plugin_dir = ( - REPO_ROOT / "plugins" / "rust" / "python-package" / "pii_filter" - ) - package_dir = plugin_dir / "cpex_pii_filter" - if not self._source_tree_has_extension(package_dir, "pii_filter_rust"): - self.skipTest("pii_filter extension is not built in the source tree") - - result = subprocess.run( + with self.subTest(paths=paths): + result = self.selection(paths) + self.assertEqual(result["mutation_jobs"], expected) + self.assertEqual(result["has_mutation_cargo_packages"], bool(expected)) + + def test_shared_crate_mutation_runs_only_its_rust_dependents(self) -> None: + manifest = self.root / RUST_PATH / "Cargo.toml" + manifest.write_text(manifest.read_text() + "[dependencies]\ncpex_framework_bridge = { workspace = true }\n") + result = self.selection(["crates/framework_bridge/src/lib.rs"]) + self.assertEqual(result["plugins"], ALL_PLUGINS) + self.assertEqual( + result["mutation_jobs"], [ - "python3", - "-c", - "import cpex_pii_filter; print(hasattr(cpex_pii_filter, 'PIIDetectorRust'))", + {"cargo_package": "cpex_framework_bridge", "in_diff": True, "test_packages": ["rust_demo"]}, ], - cwd=plugin_dir, - text=True, - capture_output=True, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "True") - - def test_pii_stub_examples_use_canonical_import_path(self) -> None: - stub_path = ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "pii_filter" - / "cpex_pii_filter" - / "pii_filter_rust" - / "__init__.pyi" ) - text = stub_path.read_text() - self.assertIn("from cpex_pii_filter import PIIDetectorRust", text) - self.assertNotIn("from pii_filter import PIIDetectorRust", text) - def test_pii_benchmark_script_runs_with_package_surface(self) -> None: - plugin_dir = ( - REPO_ROOT / "plugins" / "rust" / "python-package" / "pii_filter" - ) - package_dir = plugin_dir / "cpex_pii_filter" - if not self._source_tree_has_extension(package_dir, "pii_filter_rust"): - self.skipTest("pii_filter extension is not built in the source tree") - env = dict(os.environ) - existing = env.get("PYTHONPATH") - plugin_path = str(plugin_dir) - tests_path = str(plugin_dir / "tests") - env["PYTHONPATH"] = ( - f"{plugin_path}:{tests_path}:{existing}" - if existing - else f"{plugin_path}:{tests_path}" +class ReleaseTests(CatalogTestCase): + def test_resolves_canonical_tags_for_each_language(self) -> None: + for tag, slug, language, path in ( + ("rust-demo-v1.0.0", "rust_demo", "rust", RUST_PATH), + ("python-demo-v1.0.0", "python_demo", "python", PYTHON_PATH), + ): + with self.subTest(tag=tag): + record = catalog.release_info(self.root, tag) + self.assertEqual((record.slug, record.language, record.path, record.version), (slug, language, path, "1.0.0")) + + def test_rejects_unknown_noncanonical_and_wrong_version_tags(self) -> None: + for tag in ("v1.0.0", "unknown-v1.0.0", "rust_demo-v1.0.0", "rust-demo-v2.0.0"): + with self.subTest(tag=tag), self.assertRaises(catalog.CatalogError): + catalog.release_info(self.root, tag) + + +class GitIntegrationTests(CatalogTestCase): + def setUp(self) -> None: + super().setUp() + self.git("init") + self.git("config", "user.name", "Catalog Tests") + self.git("config", "user.email", "catalog@example.test") + self.git("config", "commit.gpgsign", "false") + self.git("config", "core.hooksPath", str(self.root / "no-hooks")) + self.commit() + self.base = self.git("rev-parse", "HEAD") + + def git(self, *args: str) -> str: + return subprocess.run(["git", *args], cwd=self.root, text=True, capture_output=True, check=True).stdout.strip() + + def commit(self) -> None: + self.git("add", ".") + self.git("commit", "-s", "-m", "Catalog fixture") + + def test_diff_cli_routes_changes_and_version_bumps_for_both_languages(self) -> None: + for path, slug, filename in ((RUST_PATH, "rust_demo", "Cargo.toml"), (PYTHON_PATH, "python_demo", "pyproject.toml")): + version_file = self.root / path / filename + version_file.write_text(version_file.read_text().replace('"1.0.0"', '"1.1.0"')) + manifest = self.root / path / f"cpex_{slug}/plugin-manifest.yaml" + manifest.write_text(manifest.read_text().replace('"1.0.0"', '"1.1.0"')) + source = self.root / RUST_PATH / "src/lib.rs" + source.parent.mkdir() + source.write_text("// changed Rust source\n") + self.commit() + + changed = self.run_catalog("changed", self.base, "HEAD") + self.assertEqual(changed.returncode, 0, changed.stderr) + self.assertEqual(json.loads(changed.stdout), {"plugins": ["python_demo", "rust_demo"]}) + selection = self.run_catalog("ci-selection", "diff", self.base, "HEAD") + self.assertEqual(selection.returncode, 0, selection.stderr) + payload = json.loads(selection.stdout) + self.assertEqual(payload["cargo_packages"], ["rust_demo"]) + self.assertEqual(payload["rust_release_validation_tags"], ["rust-demo-v1.1.0"]) + self.assertEqual(payload["python_release_validation_tags"], ["python-demo-v1.1.0"]) + self.assertEqual(payload["release_validation_tags"], ["python-demo-v1.1.0", "rust-demo-v1.1.0"]) + self.assertEqual(payload["mutation_cargo_packages"], ["rust_demo"]) + + def test_new_plugin_is_validated_for_initial_release(self) -> None: + self.create_plugin("new_demo", "python") + workspace = self.root / "pyproject.toml" + workspace.write_text(f'[tool.uv.workspace]\nmembers = ["{PYTHON_PATH}", "plugins/python/new_demo"]\n') + self.commit() + result = catalog.ci_selection(self.root, "diff", self.base, "HEAD") + self.assertEqual(result["python_release_validation_tags"], ["new-demo-v1.0.0"]) + + def test_unchanged_version_does_not_trigger_release(self) -> None: + manifest = self.root / RUST_PATH / "Cargo.toml" + manifest.write_text(manifest.read_text() + "# metadata edit\n") + self.commit() + result = catalog.ci_selection(self.root, "diff", self.base, "HEAD") + self.assertEqual(result["plugins"], ["rust_demo"]) + self.assertEqual(result["release_validation_tags"], []) + + +class CoverageTests(CatalogTestCase): + def report(self, entries: dict[str, list[str]]) -> Path: + classes = [] + for filename, hits in entries.items(): + lines = "".join(f'' for number, hit in enumerate(hits, 1)) + classes.append(f'{lines}') + report = self.root / "coverage.xml" + report.write_text("" + "".join(classes) + "") + return report + + def test_counts_each_rust_plugin_and_ignores_non_plugin_code(self) -> None: + report = self.report( + { + f"{RUST_PATH}/src/lib.rs": ["1", "0"], + r"C:\repo\plugins\rust\python-package\other_demo\src\lib.rs": ["1", "1", "1", "0"], + "crates/framework_bridge/src/lib.rs": ["0"], + f"{PYTHON_PATH}/cpex_python_demo/plugin.py": ["0"], + } ) - result = subprocess.run( - [ - "python3", - "benchmarks/compare_pii_filter.py", - "--iterations", - "1", - ], - cwd=plugin_dir, - env=env, - text=True, - capture_output=True, - check=False, + payload = catalog.coverage_check(self.root, report, 50) + self.assertEqual( + payload["plugins"], + { + "rust_demo": {"covered_lines": 1, "valid_lines": 2, "line_rate": 50.0}, + "other_demo": {"covered_lines": 3, "valid_lines": 4, "line_rate": 75.0}, + }, ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("ops_per_sec", result.stdout) + self.assertEqual(payload["minimum_plugin"], "rust_demo") + self.assertEqual(payload["minimum_line_rate"], 50.0) + with self.assertRaisesRegex(catalog.CatalogError, "below 50.10%"): + catalog.coverage_check(self.root, report, 50.1) - def test_bench_no_run_uses_nextest_benchmark_test_mode(self) -> None: - plugin_root = REPO_ROOT / "plugins" / "rust" / "python-package" - for slug in ( - "encoded_exfil_detection", - "pii_filter", - "rate_limiter", - "secrets_detection", - "url_reputation", - ): - makefile = (plugin_root / slug / "Makefile").read_text() - self.assertIn("bench-no-run:", makefile) - self.assertIn("$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run", makefile) - self.assertNotIn("$(CARGO) bench --no-run", makefile) - - def test_nextest_make_targets_dry_run(self) -> None: - plugin_root = REPO_ROOT / "plugins" / "rust" / "python-package" + def test_rejects_incomplete_or_invalid_coverage(self) -> None: cases = [ - ("encoded_exfil_detection", "test-unit", "cargo nextest run --profile default -p encoded_exfil_detection"), - ("encoded_exfil_detection", "bench-no-run", "cargo nextest run --profile default -p encoded_exfil_detection --benches -E 'kind(bench)' --no-run"), - ("pii_filter", "test-unit", "cargo nextest run --profile default -p pii_filter"), - ("pii_filter", "bench-no-run", "cargo nextest run --profile default -p pii_filter --benches -E 'kind(bench)' --no-run"), - ("rate_limiter", "test-unit", "cargo nextest run --profile default -p rate_limiter"), - ("rate_limiter", "bench-no-run", "cargo nextest run --profile default -p rate_limiter --benches -E 'kind(bench)' --no-run"), - ("retry_with_backoff", "test-unit", "cargo nextest run --profile default -p retry_with_backoff"), - ("secrets_detection", "test-unit", "cargo nextest run --profile default -p secrets_detection"), - ("secrets_detection", "bench-no-run", "cargo nextest run --profile default -p secrets_detection --benches -E 'kind(bench)' --no-run"), - ("url_reputation", "test-unit", "cargo nextest run --profile default -p url_reputation"), - ("url_reputation", "bench-no-run", "cargo nextest run --profile default -p url_reputation --benches -E 'kind(bench)' --no-run"), + ({f"{RUST_PATH}/src/lib.rs": ["1"]}, None, "missing plugin coverage"), + ({f"{RUST_PATH}/src/lib.rs": ["1"]}, ["unknown"], "Unknown expected"), + ({"plugins/rust/python-package/unknown/src/lib.rs": ["1"]}, None, "unknown plugin coverage"), + ({f"{RUST_PATH}/src/lib.rs": ["1"], f"{OTHER_PATH}/src/lib.rs": ["1"]}, ["rust_demo"], "unexpected plugin coverage"), + ({f"{RUST_PATH}/src/lib.rs": []}, ["rust_demo"], "no counted coverage lines"), + ({f"{RUST_PATH}/src/lib.rs": ["NaN"]}, ["rust_demo"], "Invalid coverage hit count"), ] - for slug, target, expected in cases: - with self.subTest(slug=slug, target=target): - result = subprocess.run( - ["make", "-n", target], - cwd=plugin_root / slug, - text=True, - capture_output=True, - check=False, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn(expected, result.stdout) - - def test_scaffold_benchmark_template_uses_nextest_benchmark_test_mode(self) -> None: - makefile_template = (REPO_ROOT / "tools" / "templates" / "plugin" / "Makefile.j2").read_text() - self.assertIn("bench-no-run:", makefile_template) - self.assertIn("$(CARGO) nextest run --profile $(NEXTEST_PROFILE) -p $(CARGO_PACKAGE) --benches -E 'kind(bench)' --no-run", makefile_template) - self.assertIn("ci: check-all verify-stubs build{% if include_benchmarks %} bench-no-run{% endif %} install-wheel test-python", makefile_template) - - def test_root_plugin_test_uses_plugin_ci_target(self) -> None: - makefile = (REPO_ROOT / "Makefile").read_text() - self.assertIn("make ci", makefile) - self.assertNotIn("make install && make test-all", makefile) - - def test_root_makefile_exposes_plugin_mutants_targets(self) -> None: - makefile = (REPO_ROOT / "Makefile").read_text() - self.assertIn("plugin-mutants PLUGIN=", makefile) - self.assertIn("plugin-mutants-list PLUGIN=", makefile) - self.assertIn('cargo mutants -p "$(PLUGIN)"', makefile) - self.assertIn('cargo mutants --list -p "$(PLUGIN)"', makefile) - - def test_retry_make_ci_enforces_local_coverage_floor(self) -> None: - plugin_dir = REPO_ROOT / "plugins" / "rust" / "python-package" / "retry_with_backoff" - makefile = (plugin_dir / "Makefile").read_text() - self.assertIn("ci: ci-build test-integration coverage", makefile) - self.assertIn("rustup component add llvm-tools-preview", makefile) - self.assertIn("cargo install cargo-llvm-cov --version $(CARGO_LLVM_COV_VERSION) --locked", makefile) - self.assertIn("cargo llvm-cov clean --workspace", makefile) - self.assertIn("$(CARGO) llvm-cov nextest --no-report -p $(CARGO_PACKAGE) -P $(NEXTEST_PROFILE)", makefile) - self.assertIn("cargo llvm-cov report -p $(CARGO_PACKAGE)", makefile) - self.assertIn( - "uv run python $(REPO_ROOT)/tools/plugin_catalog.py coverage-check $(REPO_ROOT) $(COVERAGE_REPORT) $(COVERAGE_MIN)", - makefile, - ) - - def test_secrets_detection_keeps_scanner_module_internal(self) -> None: - lib_rs = ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "secrets_detection" - / "src" - / "lib.rs" - ).read_text() - bench_rs = ( - REPO_ROOT - / "plugins" - / "rust" - / "python-package" - / "secrets_detection" - / "benches" - / "secrets_detection.rs" - ).read_text() - self.assertIn("mod scanner;", lib_rs) - self.assertNotIn("pub mod scanner;", lib_rs) - self.assertIn("pub use scanner::{detect_and_redact, scan_container};", lib_rs) - self.assertIn("use secrets_detection_rust::detect_and_redact;", bench_rs) - self.assertNotIn("use secrets_detection_rust::scanner::detect_and_redact;", bench_rs) - - def test_python_plugin_discovery_reports_typed_language_records(self) -> None: - # Given a repository containing one Rust and one pure-Python plugin. - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - rust_path = "plugins/rust/python-package/rust_demo" - (root / "Cargo.toml").write_text( - f'[workspace]\nmembers = ["{rust_path}"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "rust_demo") - self._create_python_plugin(root, "python_demo") - - # When the real catalog lists managed plugins. - result = run_catalog("list", str(root)) - - # Then both roots are discovered with language-specific Cargo metadata. - self.assertEqual(result.returncode, 0, result.stderr) - discovered = json.loads(result.stdout)["plugins"] - self.assertEqual( - [record["slug"] for record in discovered], ["rust_demo", "python_demo"] - ) - records = {record["slug"]: record for record in discovered} - self.assertEqual(records["rust_demo"]["language"], "rust") - self.assertEqual(records["rust_demo"]["cargo_package_name"], "rust_demo") - self.assertEqual(records["python_demo"]["language"], "python") - self.assertIsNone(records["python_demo"]["cargo_package_name"]) - - def test_discovery_rejects_duplicate_slug_across_managed_roots(self) -> None: - # Given valid Rust and pure-Python plugins with the same slug. - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - slug = "shared_demo" - rust_path = f"plugins/rust/python-package/{slug}" - (root / "Cargo.toml").write_text( - f'[workspace]\nmembers = ["{rust_path}"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, slug) - self._create_python_plugin(root, slug) - - # When discovery crosses the second managed root, then ambiguity is rejected. - with self.assertRaisesRegex(CatalogError, slug) as raised: - discover_plugins(root) - - self.assertIn("across managed roots", str(raised.exception)) - self.assertIn(rust_path, str(raised.exception)) - self.assertIn(f"plugins/python/{slug}", str(raised.exception)) - - def test_python_plugin_validates_without_cargo_workspace_membership(self) -> None: - # Given a valid Python plugin that is only a root uv workspace member. - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - self._create_python_plugin(root, "python_demo") - - # When catalog validation runs. - result = run_catalog("validate", str(root)) - - # Then the absent Cargo workspace membership is accepted. - self.assertEqual(result.returncode, 0, result.stderr) - - def test_python_validator_rejects_cargo_workspace_membership(self) -> None: - # Given a Python plugin incorrectly added to the Cargo workspace. - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - self._create_python_plugin(root, "python_demo") - (root / "Cargo.toml").write_text( - '[workspace]\nmembers = ["plugins/python/python_demo"]\n' - ) - - # When validation runs. - result = run_catalog("validate", str(root)) - - # Then language ownership prevents Cargo from claiming the Python plugin. - self.assertNotEqual(result.returncode, 0) - self.assertIn("must not be cargo workspace members", result.stderr.lower()) - - def test_python_validator_rejects_invalid_project_metadata(self) -> None: - cases = { - "static project.version": ( - '[project]\nname = "cpex-python-demo"\n', - True, - ), - "dynamic": ( - ( - '[project]\nname = "cpex-python-demo"\nversion = "0.2.0"\n' - 'dynamic = ["version"]\n' - ), - True, - ), - "package name": ( - '[project]\nname = "wrong-name"\nversion = "0.2.0"\n', - False, - ), - "entry-points": ( - '[project]\nname = "cpex-python-demo"\nversion = "0.2.0"\n', - False, - ), - } - for expected_error, (project_table, include_entry_point) in cases.items(): - with self.subTest( - expected_error=expected_error - ), tempfile.TemporaryDirectory() as tmpdir: - # Given a Python fixture with one invalid project metadata field. - root = Path(tmpdir) - plugin_dir = self._create_python_plugin(root, "python_demo") - suffix = "" - if include_entry_point: - suffix = ( - '\n[project.entry-points."cpex.plugins"]\n' - 'python_demo = "cpex_python_demo.plugin:PythonDemoPlugin"\n' - ) - (plugin_dir / "pyproject.toml").write_text(project_table + suffix) - - # When validation runs. - result = run_catalog("validate", str(root)) - - # Then malformed Python project metadata is rejected. - self.assertNotEqual(result.returncode, 0) - self.assertIn(expected_error, result.stderr.lower()) - - def test_python_validator_rejects_manifest_version_mismatch(self) -> None: - # Given a Python plugin whose manifest is 0.1.0 but project version is 0.2.0. - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - plugin_dir = self._create_python_plugin(root, "python_demo", version="0.2.0") - manifest = plugin_dir / "cpex_python_demo" / "plugin-manifest.yaml" - manifest.write_text(manifest.read_text().replace('version: "0.2.0"', 'version: "0.1.0"')) - - # When validation runs. - result = run_catalog("validate", str(root)) - - # Then the cross-file version mismatch is rejected. - self.assertNotEqual(result.returncode, 0) - self.assertIn("version mismatch", result.stderr.lower()) - - def test_python_validator_rejects_noncanonical_manifest_kind(self) -> None: - # Given a Python plugin manifest using module:object instead of module.object. - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - plugin_dir = self._create_python_plugin(root, "python_demo") - manifest = plugin_dir / "cpex_python_demo" / "plugin-manifest.yaml" - manifest.write_text( - manifest.read_text().replace( - "cpex_python_demo.plugin.PythonDemoPlugin", - "cpex_python_demo.plugin:PythonDemoPlugin", - ) - ) - - # When validation runs. - result = run_catalog("validate", str(root)) - - # Then the established canonical kind rule is preserved. - self.assertNotEqual(result.returncode, 0) - self.assertIn("module.object", result.stderr.lower()) - - def test_python_validator_requires_root_uv_workspace_membership(self) -> None: - # Given a Python plugin omitted from root [tool.uv.workspace].members. - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - self._create_python_plugin(root, "python_demo", add_workspace_member=False) + for entries, expected, error in cases: + with self.subTest(error=error): + report = self.report(entries) + with self.assertRaisesRegex(catalog.CatalogError, error): + catalog.coverage_check(self.root, report, 50, expected) - # When validation runs. - result = run_catalog("validate", str(root)) - # Then the missing uv workspace member is rejected. - self.assertNotEqual(result.returncode, 0) - self.assertIn("uv workspace", result.stderr.lower()) - - def test_mixed_change_routing_handles_both_plugin_roots_and_safe_test_fallback(self) -> None: - cases = { - "plugins/python/python_demo/README.md": ["python_demo"], - "plugins/tests/python_demo/test_plugin.py": ["python_demo"], - "plugins/tests/unknown_plugin/test_plugin.py": ["python_demo", "rust_demo"], - } - for changed_path, expected in cases.items(): - with self.subTest( - changed_path=changed_path - ), tempfile.TemporaryDirectory() as tmpdir: - # Given a committed mixed-language plugin repository. - root = Path(tmpdir) - rust_path = "plugins/rust/python-package/rust_demo" - (root / "Cargo.toml").write_text( - f'[workspace]\nmembers = ["{rust_path}"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "rust_demo") - self._create_python_plugin(root, "python_demo") - subprocess.run(["git", "init"], cwd=root, check=True, capture_output=True) - subprocess.run(["git", "config", "user.name", "Test User"], cwd=root, check=True) - subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=root, check=True) - subprocess.run(["git", "add", "."], cwd=root, check=True) - subprocess.run( - ["git", "commit", "--no-verify", "-m", "seed layout"], - cwd=root, - check=True, - capture_output=True, - ) - base = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=root, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - changed_file = root / changed_path - changed_file.parent.mkdir(parents=True, exist_ok=True) - changed_file.write_text("# changed\n") - subprocess.run(["git", "add", "."], cwd=root, check=True) - subprocess.run( - ["git", "commit", "--no-verify", "-m", "change route"], - cwd=root, - check=True, - capture_output=True, - ) - - # When diff selection runs. - result = run_catalog("ci-selection", str(root), "diff", base, "HEAD") - - # Then direct paths are scoped and unknown integration slugs select all. +class CliTests(CatalogTestCase): + def test_json_commands_and_scalar_fields(self) -> None: + records = catalog.discover_plugins(self.root) + cases = [ + (("validate",), {"status": "ok"}), + (("list",), {"plugins": [asdict(record) for record in records]}), + (("ci-selection", "all"), catalog.ci_selection(self.root, "all")), + (("ci-selection-field", "all", "", "", "plugins"), ALL_PLUGINS), + (("ci-selection-field", "all", "", "", "has_plugins"), True), + (("ci-selection-field", "all", "", "", "plugin_count"), 3), + ] + for args, expected in cases: + with self.subTest(args=args): + result = self.run_catalog(*args) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(json.loads(result.stdout)["plugins"], expected) - - def test_mixed_framework_bridge_change_emits_only_rust_mutation_packages(self) -> None: - # Given a mixed repository where the Rust plugin depends on framework_bridge. - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - rust_path = "plugins/rust/python-package/rust_demo" - (root / "Cargo.toml").write_text( - f'[workspace]\nmembers = ["{rust_path}"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - rust_plugin = self._create_plugin(root, "rust_demo") - (rust_plugin / "Cargo.toml").write_text( - '[package]\nname = "rust_demo"\nversion = "0.0.1"\n' - 'repository = "https://github.com/IBM/cpex-plugins"\n\n' - '[dependencies]\ncpex_framework_bridge = { workspace = true }\n' - ) - self._create_python_plugin(root, "python_demo") - bridge = root / "crates" / "framework_bridge" / "src" / "lib.rs" - bridge.parent.mkdir(parents=True) - bridge.write_text("// seed\n") - subprocess.run(["git", "init"], cwd=root, check=True, capture_output=True) - subprocess.run(["git", "config", "user.name", "Test User"], cwd=root, check=True) - subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=root, check=True) - subprocess.run(["git", "add", "."], cwd=root, check=True) - subprocess.run( - ["git", "commit", "--no-verify", "-m", "seed layout"], - cwd=root, - check=True, - capture_output=True, - ) - base = subprocess.run( - ["git", "rev-parse", "HEAD"], cwd=root, check=True, capture_output=True, text=True - ).stdout.strip() - bridge.write_text("// changed\n") - subprocess.run(["git", "add", "."], cwd=root, check=True) - subprocess.run( - ["git", "commit", "--no-verify", "-m", "bridge change"], - cwd=root, - check=True, - capture_output=True, - ) - - # When mutation selection processes the shared Rust crate change. - result = run_catalog("ci-selection", str(root), "diff", base, "HEAD") - - # Then Python records never reach Cargo dependency or mutation payloads. - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["cargo_packages"], ["rust_demo"]) - self.assertEqual(payload["mutation_jobs"][0]["test_packages"], ["rust_demo"]) - - def test_python_version_bump_emits_python_release_validation_tag(self) -> None: - # Given a committed Python plugin bumped from 0.1.0 to 0.2.0. - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - plugin = self._create_python_plugin(root, "python_demo", version="0.1.0") - subprocess.run(["git", "init"], cwd=root, check=True, capture_output=True) - subprocess.run(["git", "config", "user.name", "Test User"], cwd=root, check=True) - subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=root, check=True) - subprocess.run(["git", "add", "."], cwd=root, check=True) - subprocess.run( - ["git", "commit", "--no-verify", "-m", "seed layout"], - cwd=root, - check=True, - capture_output=True, - ) - base = subprocess.run( - ["git", "rev-parse", "HEAD"], cwd=root, check=True, capture_output=True, text=True - ).stdout.strip() - pyproject = plugin / "pyproject.toml" - pyproject.write_text(pyproject.read_text().replace('version = "0.1.0"', 'version = "0.2.0"')) - manifest = plugin / "cpex_python_demo" / "plugin-manifest.yaml" - manifest.write_text(manifest.read_text().replace('version: "0.1.0"', 'version: "0.2.0"')) - subprocess.run(["git", "add", "."], cwd=root, check=True) - subprocess.run( - ["git", "commit", "--no-verify", "-m", "bump version"], - cwd=root, - check=True, - capture_output=True, - ) - - # When diff selection computes release validation tags. - result = run_catalog("ci-selection", str(root), "diff", base, "HEAD") - - # Then the Python pyproject version drives both aggregate and split tags. - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["release_validation_tags"], ["python-demo-v0.2.0"]) - self.assertEqual(payload["python_release_validation_tags"], ["python-demo-v0.2.0"]) - self.assertTrue(payload["has_python_release_validation_tags"]) - - def test_python_release_info_reports_language_and_path(self) -> None: - # Given a valid Python plugin. - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - self._create_python_plugin(root, "python_demo") - - # When release info resolves its canonical tag. - result = run_catalog("release-info", str(root), "python-demo-v0.2.0") - - # Then release metadata identifies the Python catalog source. - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["language"], "python") - self.assertEqual(payload["path"], "plugins/python/python_demo") - - def test_ci_selection_reports_language_splits_and_counts(self) -> None: - # Given the repository and a mixed fixture repository. - result = run_catalog("ci-selection", str(REPO_ROOT), "all", "", "") + self.assertEqual(json.loads(result.stdout), expected) + result = self.run_catalog("release-info-field", "python-demo-v1.0.0", "kind") self.assertEqual(result.returncode, 0, result.stderr) - repository_payload = json.loads(result.stdout) - - # Then all-mode characterizes the required 8 Rust / 1 Python split. - self.assertEqual(repository_payload["rust_plugin_count"], 8) - self.assertEqual(repository_payload["python_plugin_count"], 1) - self.assertTrue(repository_payload["has_rust_plugins"]) - self.assertTrue(repository_payload["has_python_plugins"]) + self.assertEqual(result.stdout.strip(), "cpex_python_demo.plugin.DemoPlugin") - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - rust_path = "plugins/rust/python-package/rust_demo" - (root / "Cargo.toml").write_text( - f'[workspace]\nmembers = ["{rust_path}"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "rust_demo") - self._create_python_plugin(root, "python_demo") + def test_errors_exit_cleanly_without_tracebacks(self) -> None: + (self.root / RUST_PATH / "pyproject.toml").write_text("[project") + result = self.run_catalog("validate") + self.assertEqual(result.returncode, 1) + self.assertIn("Invalid TOML", result.stderr) + self.assertNotIn("Traceback", result.stderr) + self.assertEqual(result.stdout, "") - # When mixed all-mode selection runs. - result = run_catalog("ci-selection", str(root), "all", "", "") - # Then aggregate compatibility fields and language splits agree. - self.assertEqual(result.returncode, 0, result.stderr) - payload = json.loads(result.stdout) - self.assertEqual(payload["plugins"], ["python_demo", "rust_demo"]) - self.assertEqual(payload["rust_plugins"], ["rust_demo"]) - self.assertEqual(payload["python_plugins"], ["python_demo"]) - self.assertEqual(payload["rust_plugin_count"], 1) - self.assertEqual(payload["python_plugin_count"], 1) - self.assertEqual(payload["cargo_packages"], ["rust_demo"]) - self.assertEqual(payload["mutation_cargo_packages"], ["rust_demo"]) - self.assertEqual(payload["mutation_jobs"][0]["cargo_package"], "rust_demo") - - def test_ci_selection_validator_reemits_every_split_field(self) -> None: - # Given a complete valid payload containing aggregate and split fields. - payload = { - "plugins": ["python_demo", "rust_demo"], - "rust_plugins": ["rust_demo"], - "python_plugins": ["python_demo"], - "has_plugins": True, - "has_rust_plugins": True, - "has_python_plugins": True, - "plugin_count": 2, - "rust_plugin_count": 1, - "python_plugin_count": 1, - "cargo_packages": ["rust_demo"], - "mutation_cargo_packages": [], - "has_mutation_cargo_packages": False, - "mutation_jobs": [], - "release_validation_tags": ["python-demo-v0.2.0"], - "rust_release_validation_tags": [], - "python_release_validation_tags": ["python-demo-v0.2.0"], - "has_release_validation_tags": True, - "has_rust_release_validation_tags": False, - "has_python_release_validation_tags": True, - } - - # When the real validator parses and normalizes it. - result = subprocess.run( - ["python3", str(CI_SELECTION_VALIDATOR)], +class SelectionValidatorTests(CatalogTestCase): + def validate(self, payload: dict) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SELECTION_VALIDATOR)], input=json.dumps(payload), text=True, capture_output=True, check=False, ) - # Then every field survives strict validation unchanged. + def test_reemits_complete_catalog_selection(self) -> None: + payload = catalog.ci_selection(self.root, "all") + result = self.validate(payload) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(json.loads(result.stdout), payload) - def test_ci_selection_validator_rejects_missing_malformed_and_mismatched_split_fields(self) -> None: - valid_payload = { - "plugins": ["python_demo"], - "rust_plugins": [], - "python_plugins": ["python_demo"], - "has_plugins": True, - "has_rust_plugins": False, - "has_python_plugins": True, - "plugin_count": 1, - "rust_plugin_count": 0, - "python_plugin_count": 1, - "cargo_packages": [], - "mutation_cargo_packages": [], - "has_mutation_cargo_packages": False, - "mutation_jobs": [], - "release_validation_tags": [], - "rust_release_validation_tags": [], - "python_release_validation_tags": [], - "has_release_validation_tags": False, - "has_rust_release_validation_tags": False, - "has_python_release_validation_tags": False, - } - malformed_payloads = { - "missing": {key: value for key, value in valid_payload.items() if key != "python_plugins"}, - "wrong type": {**valid_payload, "has_python_plugins": "true"}, - "malformed slug": {**valid_payload, "python_plugins": ["python-demo"]}, - "count mismatch": {**valid_payload, "python_plugin_count": 2}, - } - for case, payload in malformed_payloads.items(): - with self.subTest(case=case): - # When malformed input crosses the validator boundary. - result = subprocess.run( - ["python3", str(CI_SELECTION_VALIDATOR)], - input=json.dumps(payload), - text=True, - capture_output=True, - check=False, - ) - - # Then it is rejected rather than normalized into misleading success. + def test_rejects_malformed_and_inconsistent_job_inputs(self) -> None: + cases = [ + ("python_plugins", None), + ("plugins", ["../invalid"]), + ("has_plugins", "true"), + ("has_plugins", False), + ("plugin_count", 99), + ("plugin_count", True), + ("python_plugins", ["other_plugin"]), + ("release_validation_tags", ["python-demo-v1.0.0"]), + ("mutation_jobs", [{"cargo_package": "rust_demo", "in_diff": "true", "test_packages": []}]), + ] + for field, value in cases: + with self.subTest(field=field, value=value): + payload = catalog.ci_selection(self.root, "all") + if value is None: + del payload[field] + else: + payload[field] = value + result = self.validate(payload) self.assertNotEqual(result.returncode, 0) - - def test_coverage_default_expected_plugins_remains_rust_only(self) -> None: - # Given mixed discovery and a Rust-only coverage report. - with tempfile.TemporaryDirectory() as tmpdir: - root = Path(tmpdir) - rust_path = "plugins/rust/python-package/rust_demo" - (root / "Cargo.toml").write_text( - f'[workspace]\nmembers = ["{rust_path}"]\n' - '[workspace.package]\nrepository = "https://github.com/IBM/cpex-plugins"\n' - ) - self._create_plugin(root, "rust_demo") - self._create_python_plugin(root, "python_demo") - report = root / "coverage.xml" - report.write_text( - '' - '' - '' - '' - ) - - # When coverage-check uses its discovered default expectation. - result = run_catalog("coverage-check", str(root), str(report), "100") - - # Then a pure-Python plugin is not required in Rust coverage XML. - self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "") if __name__ == "__main__": diff --git a/tests/test_repository_policy.py b/tests/test_repository_policy.py new file mode 100644 index 00000000..fc0ff168 --- /dev/null +++ b/tests/test_repository_policy.py @@ -0,0 +1,35 @@ +# Copyright 2026 +# SPDX-License-Identifier: Apache-2.0 +"""Supply-chain policies that build and catalog validation do not enforce.""" + +import re +import tomllib +import unittest +from datetime import UTC, date, datetime +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +class RepositoryPolicyTests(unittest.TestCase): + def test_external_workflow_actions_are_pinned_to_commit_shas(self) -> None: + for workflow in (REPO_ROOT / ".github/workflows").glob("*.y*ml"): + for number, line in enumerate(workflow.read_text().splitlines(), 1): + match = re.match(r"^\s*-?\s*uses:\s+(\S+)", line) + if match is None: + continue + reference = match[1].strip("\"'") + if reference.startswith("./"): + continue + with self.subTest(workflow=workflow.name, line=number): + self.assertRegex(reference, r"^[^@]+@[0-9a-f]{40}$") + + def test_cargo_advisory_exceptions_have_not_expired(self) -> None: + config = tomllib.loads((REPO_ROOT / "deny.toml").read_text()) + for entry in config.get("advisories", {}).get("ignore", []): + if not isinstance(entry, dict): + continue + match = re.search(r"expires (\d{4}-\d{2}-\d{2})", entry.get("reason", "")) + if match: + with self.subTest(advisory=entry.get("id")): + self.assertLessEqual(datetime.now(UTC).date(), date.fromisoformat(match[1])) diff --git a/tools/templates/plugin/README.md.j2 b/tools/templates/plugin/README.md.j2 index 706cfc5f..06348d62 100644 --- a/tools/templates/plugin/README.md.j2 +++ b/tools/templates/plugin/README.md.j2 @@ -60,8 +60,8 @@ flowchart LR ## TODOs - [ ] Implement core functionality in `src/engine.rs` -- [ ] Add comprehensive unit tests -- [ ] Add integration tests +- [ ] Test the plugin's decisions and meaningful failure modes in Rust +- [ ] Extend the Python hook smoke tests for boundary behavior {% if include_benchmarks -%} - [ ] Add performance benchmarks {% endif -%} diff --git a/tools/templates/plugin/tests/test_plugin.py.j2 b/tools/templates/plugin/tests/test_plugin.py.j2 index e1ddff3d..4e7c395c 100644 --- a/tools/templates/plugin/tests/test_plugin.py.j2 +++ b/tools/templates/plugin/tests/test_plugin.py.j2 @@ -1,129 +1,29 @@ # SPDX-License-Identifier: Apache-2.0 -# Copyright 2024 ContextForge Contributors -"""Tests for {{ plugin_name_title }} plugin.""" - -import pytest - -from {{ module_name }} import {{ config_class }}, {{ engine_class }}, {{ plugin_class }} - - -class Test{{ engine_class }}: - """Tests for {{ engine_class }}.""" - - def test_engine_creation(self): - """Test engine can be created with default config.""" - config = {{ config_class }}() - engine = {{ engine_class }}(config.model_dump()) - assert engine is not None - - def test_engine_with_custom_config(self): - """Test engine can be created with custom config.""" - config = {{ config_class }}(example_option="custom_value") - engine = {{ engine_class }}(config.model_dump()) - assert engine is not None - - {% for hook in hooks %} - {% if hook == "tool_pre_invoke" %} - def test_tool_pre_invoke_allows_by_default(self): - """Test tool_pre_invoke allows execution by default.""" - config = {{ config_class }}() - engine = {{ engine_class }}(config.model_dump()) - # TODO: Create proper test payload and context - # result = engine.tool_pre_invoke(payload, context) - # assert result.continue_processing is True - pass - {% elif hook == "resource_pre_fetch" %} - def test_resource_pre_fetch_allows_by_default(self): - """Test resource_pre_fetch allows fetch by default.""" - config = {{ config_class }}() - engine = {{ engine_class }}(config.model_dump()) - # TODO: Create proper test payload and context - # result = engine.resource_pre_fetch(payload, context) - # assert result.continue_processing is True - pass - {% endif %} - {% endfor %} - - -class Test{{ plugin_class }}: - """Tests for {{ plugin_class }}.""" - - def test_plugin_requires_rust_module(self): - """Test plugin raises error if Rust module not available.""" - # This test assumes the module is installed - # In a real scenario, you'd mock the import - pass - - def test_plugin_creation_with_config(self): - """Test plugin can be created with configuration.""" - # TODO: Create proper plugin config object - # plugin = {{ plugin_class }}(config) - # assert plugin is not None - pass +# Copyright 2026 ContextForge Contributors +"""Smoke tests for the generated Python wrapper and compiled Rust hooks. +Add decision and failure-mode tests as the plugin gains behavior. Test core +algorithms in Rust; keep Python tests focused on the packaging/hook boundary. +""" -class Test{{ config_class }}: - """Tests for {{ config_class }}.""" - - def test_config_defaults(self): - """Test configuration has correct defaults.""" - config = {{ config_class }}() - assert config.example_option == "default_value" - - def test_config_validation(self): - """Test configuration validation.""" - config = {{ config_class }}(example_option="test") - assert config.example_option == "test" - - def test_config_serialization(self): - """Test configuration can be serialized.""" - config = {{ config_class }}(example_option="test") - data = config.model_dump() - assert isinstance(data, dict) - assert data["example_option"] == "test" - - def test_config_deserialization(self): - """Test configuration can be deserialized from dict.""" - data = {"example_option": "from_dict"} - config = {{ config_class }}(**data) - assert config.example_option == "from_dict" - - def test_config_immutability(self): - """Test configuration fields are validated on creation.""" - config = {{ config_class }}(example_option="initial") - # Pydantic models are mutable by default, but validation occurs - assert config.example_option == "initial" - - def test_config_with_none_value(self): - """Test configuration handles None values appropriately.""" - # Depending on field definition, None may or may not be allowed - config = {{ config_class }}() - assert hasattr(config, "example_option") - - -class TestEdgeCases: - """Edge case tests for {{ plugin_name_title }} plugin.""" - - def test_config_with_empty_string(self): - """Test configuration handles empty string values.""" - config = {{ config_class }}(example_option="") - assert config.example_option == "" - - def test_config_with_special_characters(self): - """Test configuration handles special characters.""" - special_value = "test!@#$%^&*()" - config = {{ config_class }}(example_option=special_value) - assert config.example_option == special_value - - def test_engine_with_minimal_config(self): - """Test engine works with minimal configuration.""" - config = {{ config_class }}() - engine = {{ engine_class }}(config.model_dump()) - assert engine is not None +import pytest - def test_engine_config_roundtrip(self): - """Test engine configuration can be serialized and deserialized.""" - config = {{ config_class }}(example_option="roundtrip_test") - data = config.model_dump() - new_config = {{ config_class }}(**data) - assert new_config.example_option == config.example_option +from cpex.framework import PluginConfig +from {{ module_name }} import {{ plugin_class }} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("hook", {{ hooks }}) +async def test_hook_allows_request(hook: str) -> None: + plugin = {{ plugin_class }}( + PluginConfig( + name="{{ plugin_name }}", + kind="{{ module_name }}.{{ plugin_name }}.{{ plugin_class }}", + hooks={{ hooks }}, + config={}, + ) + ) + result = await getattr(plugin, hook)({"value": "example"}, None) + assert result.continue_processing is True + assert result.violation is None + assert "error" not in (result.metadata or {})