ci-rust-python-package.yaml

name: CI Rust Python Package Plugins

on:
  push:
    branches: [main]
    paths:
      - "Makefile"
      - "pyproject.toml"
      - "uv.lock"
      - "Cargo.toml"
      - "Cargo.lock"
      - ".cargo/**"
      - ".config/nextest.toml"
      - "deny.toml"
      - "crates/**"
      - "README.md"
      - "DEVELOPING.md"
      - "TESTING.md"
      - "plugins/rust/python-package/**"
      - "plugins/tests/**"
      - "tools/**"
      - ".github/workflows/ci-rust-python-package.yaml"
      - ".github/workflows/release-rust-python-package.yaml"
  pull_request:
    branches: [main]
    paths:
      - "Makefile"
      - "pyproject.toml"
      - "uv.lock"
      - "Cargo.toml"
      - "Cargo.lock"
      - ".cargo/**"
      - ".config/nextest.toml"
      - "deny.toml"
      - "crates/**"
      - "README.md"
      - "DEVELOPING.md"
      - "TESTING.md"
      - "plugins/rust/python-package/**"
      - "plugins/tests/**"
      - "tools/**"
      - ".github/workflows/ci-rust-python-package.yaml"
      - ".github/workflows/release-rust-python-package.yaml"
  workflow_dispatch:

concurrency:
  group: ci-rust-python-package-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }}
  # Main runs can create release tags after full CI, so only superseded PR runs are safe to cancel.
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
  contents: read
  pull-requests: read

jobs:
  validate-and-detect:
    runs-on: ubuntu-latest
    outputs:
      plugins: ${{ steps.detect.outputs.plugins }}
      has_plugins: ${{ steps.detect.outputs.has_plugins }}
      plugin_count: ${{ steps.detect.outputs.plugin_count }}
      cargo_packages: ${{ steps.detect.outputs.cargo_packages }}
      mutation_cargo_packages: ${{ steps.detect.outputs.mutation_cargo_packages }}
      mutation_jobs: ${{ steps.detect.outputs.mutation_jobs }}
      has_mutation_cargo_packages: ${{ steps.detect.outputs.has_mutation_cargo_packages }}
      release_validation_tags: ${{ steps.detect.outputs.release_validation_tags }}
      has_release_validation_tags: ${{ steps.detect.outputs.has_release_validation_tags }}
      rust_plugins: ${{ steps.detect.outputs.rust_plugins }}
      has_rust_plugins: ${{ steps.detect.outputs.has_rust_plugins }}
      rust_plugin_count: ${{ steps.detect.outputs.rust_plugin_count }}
      rust_release_validation_tags: ${{ steps.detect.outputs.rust_release_validation_tags }}
      has_rust_release_validation_tags: ${{ steps.detect.outputs.has_rust_release_validation_tags }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0

      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - id: detect
        shell: bash
        run: |
          set -euo pipefail
          if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
            selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}")"
          elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
            selection="$(python3 tools/plugin_catalog.py ci-selection . all '' '')"
          elif [[ "${{ github.event.before }}" == "0000000000000000000000000000000000000000" ]]; then
            selection="$(python3 tools/plugin_catalog.py ci-selection . all '' '')"
          else
            selection="$(python3 tools/plugin_catalog.py ci-selection . diff "${{ github.event.before }}" "${{ github.sha }}")"
          fi

          selection="$(printf '%s' "${selection}" | python3 tools/validate_ci_selection.py)"
          plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["plugins"]))')"
          has_plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_plugins"]).lower())')"
          plugin_count="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.load(sys.stdin)["plugin_count"])')"
          cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["cargo_packages"]))')"
          mutation_cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["mutation_cargo_packages"]))')"
          mutation_jobs="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["mutation_jobs"]))')"
          has_mutation_cargo_packages="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_mutation_cargo_packages"]).lower())')"
          release_validation_tags="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["release_validation_tags"]))')"
          has_release_validation_tags="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_release_validation_tags"]).lower())')"
          rust_plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["rust_plugins"]))')"
          has_rust_plugins="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_rust_plugins"]).lower())')"
          rust_plugin_count="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.load(sys.stdin)["rust_plugin_count"])')"
          rust_release_validation_tags="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(json.dumps(json.load(sys.stdin)["rust_release_validation_tags"]))')"
          has_rust_release_validation_tags="$(printf '%s' "${selection}" | python3 -c 'import json, sys; print(str(json.load(sys.stdin)["has_rust_release_validation_tags"]).lower())')"
          if [[ "${has_plugins}" == "false" ]]; then
            has_plugins_output="false"
          else
            has_plugins_output="true"
          fi
          has_mutation_cargo_packages_output="${has_mutation_cargo_packages}"
          {
            echo "plugins=${plugins}"
            echo "plugin_count=${plugin_count}"
            echo "cargo_packages=${cargo_packages}"
            echo "has_plugins=${has_plugins_output}"
            echo "mutation_cargo_packages=${mutation_cargo_packages}"
            echo "mutation_jobs=${mutation_jobs}"
            echo "has_mutation_cargo_packages=${has_mutation_cargo_packages_output}"
            echo "release_validation_tags=${release_validation_tags}"
            echo "has_release_validation_tags=${has_release_validation_tags}"
            echo "rust_plugins=${rust_plugins}"
            echo "has_rust_plugins=${has_rust_plugins}"
            echo "rust_plugin_count=${rust_plugin_count}"
            echo "rust_release_validation_tags=${rust_release_validation_tags}"
            echo "has_rust_release_validation_tags=${has_rust_release_validation_tags}"
          } >> "$GITHUB_OUTPUT"

  build-test:
    needs: validate-and-detect
    if: needs.validate-and-detect.outputs.has_rust_plugins == 'true'
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        plugin: ${{ fromJson(needs.validate-and-detect.outputs.rust_plugins) }}
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Verify Rust toolchain
        run: |
          rustc --version
          cargo --version

      - name: Install cargo-nextest
        run: |
          if [[ "${RUNNER_OS}" == "Linux" ]]; then
            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
            curl -LsSf https://get.nexte.st/0.9.133/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
          elif [[ "${RUNNER_OS}" == "macOS" ]]; then
            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
            if [[ "$(uname -m)" == "arm64" ]]; then
              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
            else
              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
            fi
          else
            cargo install cargo-nextest --version 0.9.133 --locked
          fi
          cargo nextest --version

      - name: Install uv
        run: python -m pip install uv==0.9.30 maturin==1.12.6

      - name: Sync plugin environment
        working-directory: plugins/rust/python-package/${{ matrix.plugin }}
        run: make sync

      - name: Plugin CI build verification
        if: matrix.os == 'ubuntu-latest'
        working-directory: plugins/rust/python-package/${{ matrix.plugin }}
        env:
          NEXTEST_PROFILE: ci
        run: make ci-build

      - name: Plugin CI verification
        if: matrix.os != 'ubuntu-latest'
        working-directory: plugins/rust/python-package/${{ matrix.plugin }}
        env:
          NEXTEST_PROFILE: ci
        run: make ci

  security-policy:
    needs: validate-and-detect
    if: needs.validate-and-detect.outputs.has_plugins == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Verify Rust toolchain
        run: |
          rustc --version
          cargo --version

      - name: Install cargo deny
        run: cargo install cargo-deny@0.19.0 --locked

      - name: Run cargo deny
        run: cargo deny --all-features check --config deny.toml

  mutation-testing:
    needs: validate-and-detect
    if: github.event_name == 'pull_request' && needs.validate-and-detect.outputs.has_mutation_cargo_packages == 'true'
    strategy:
      fail-fast: false
      matrix:
        mutation_job: ${{ fromJson(needs.validate-and-detect.outputs.mutation_jobs) }}
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0

      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Verify Rust toolchain
        run: |
          rustc --version
          cargo --version

      - name: Install Rust mutation testing tooling
        run: |
          if [[ "${RUNNER_OS}" == "Linux" ]]; then
            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
            curl -LsSf https://get.nexte.st/0.9.133/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
          elif [[ "${RUNNER_OS}" == "macOS" ]]; then
            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
            if [[ "$(uname -m)" == "arm64" ]]; then
              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
            else
              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
            fi
          else
            cargo install cargo-nextest --version 0.9.133 --locked
          fi
          cargo nextest --version
          cargo install cargo-mutants --version 27.0.0 --locked
          cargo mutants --version

      - name: Create mutation diff
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
        run: git diff "${BASE_SHA}" -- '*.rs' > cargo-mutants.diff

      - name: Run cargo-mutants with nextest
        env:
          CARGO_PACKAGE: ${{ matrix.mutation_job.cargo_package }}
          IN_DIFF: ${{ matrix.mutation_job.in_diff }}
          PYO3_PYTHON: python
          TEST_PACKAGES: ${{ toJson(matrix.mutation_job.test_packages) }}
        run: |
          mapfile -t test_packages < <(python3 -c 'import json, os; [print(package) for package in json.loads(os.environ["TEST_PACKAGES"])]')
          cargo_args=("-p" "${CARGO_PACKAGE}")
          if [[ "${IN_DIFF}" == "true" ]]; then
            cargo_args+=("--in-diff" "cargo-mutants.diff")
          fi
          for package in "${test_packages[@]}"; do
            cargo_args+=("--test-package" "${package}")
          done
          cargo mutants "${cargo_args[@]}"

  coverage:
    needs: validate-and-detect
    if: needs.validate-and-detect.outputs.has_rust_plugins == 'true'
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Install Rust coverage tooling
        run: |
          rustup component add llvm-tools-preview
          cargo install cargo-llvm-cov --version 0.8.4 --locked

      - name: Install cargo-nextest
        run: |
          if [[ "${RUNNER_OS}" == "Linux" ]]; then
            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
            curl -LsSf https://get.nexte.st/0.9.133/linux | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
          elif [[ "${RUNNER_OS}" == "macOS" ]]; then
            mkdir -p "${CARGO_HOME:-$HOME/.cargo}/bin"
            if [[ "$(uname -m)" == "arm64" ]]; then
              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
            else
              curl -LsSf https://get.nexte.st/0.9.133/mac | tar zxf - -C "${CARGO_HOME:-$HOME/.cargo}/bin"
            fi
          else
            cargo install cargo-nextest --version 0.9.133 --locked
          fi
          cargo nextest --version

      - name: Install Python build tooling
        run: python -m pip install uv==0.9.30 maturin==1.12.6

      - name: Generate Rust coverage report
        env:
          CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }}
          NEXTEST_PROFILE: ci
          PLUGINS: ${{ needs.validate-and-detect.outputs.rust_plugins }}
          PYO3_PYTHON: python
        run: |
          mkdir -p coverage
          mapfile -t cargo_packages < <(python3 -c 'import json, os; [print(package) for package in json.loads(os.environ["CARGO_PACKAGES"])]')
          mapfile -t plugins < <(python3 -c 'import json, os; [print(plugin) for plugin in json.loads(os.environ["PLUGINS"])]')
          cargo_args=()
          for package in "${cargo_packages[@]}"; do
            cargo_args+=("-p" "${package}")
          done
          cargo llvm-cov clean --workspace
          mkdir -p coverage
          cargo llvm-cov nextest --no-report "${cargo_args[@]}" -P "${NEXTEST_PROFILE}"
          eval "$(cargo llvm-cov show-env --sh)"
          export CARGO_TARGET_DIR="${CARGO_LLVM_COV_TARGET_DIR}/llvm-cov-target"
          export CARGO_LLVM_COV_BUILD_DIR="${CARGO_TARGET_DIR}"
          export LLVM_PROFILE_FILE="${CARGO_TARGET_DIR}/cpex-plugins-%p-%10m.profraw"
          mkdir -p "${CARGO_TARGET_DIR}"
          for plugin in "${plugins[@]}"; do
            (cd "plugins/rust/python-package/${plugin}" && make sync && uv run maturin develop)
          done
          for plugin in "${plugins[@]}"; do
            (cd "plugins/rust/python-package/${plugin}" && make test-integration)
          done
          env -u CARGO_TARGET_DIR -u CARGO_LLVM_COV_BUILD_DIR -u CARGO_LLVM_COV_TARGET_DIR -u LLVM_PROFILE_FILE cargo llvm-cov report "${cargo_args[@]}" --cobertura --output-path coverage/cobertura.xml

      - name: Enforce per-plugin coverage floor
        env:
          PLUGINS: ${{ needs.validate-and-detect.outputs.rust_plugins }}
        run: python3 tools/plugin_catalog.py coverage-check . coverage/cobertura.xml 90.00 "${PLUGINS}"

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
        with:
          files: ./coverage/cobertura.xml
          flags: rust-python-package-workspace
          name: rust-python-package-workspace-coverage

  documentation:
    needs: validate-and-detect
    if: needs.validate-and-detect.outputs.has_plugins == 'true'
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Verify Rust toolchain
        run: |
          rustc --version
          cargo --version

      - name: Build Rust documentation
        env:
          CARGO_PACKAGES: ${{ needs.validate-and-detect.outputs.cargo_packages }}
        run: |
          mapfile -t cargo_packages < <(python3 -c 'import json, os; [print(package) for package in json.loads(os.environ["CARGO_PACKAGES"])]')
          cargo_args=()
          for package in "${cargo_packages[@]}"; do
            cargo_args+=("-p" "${package}")
          done
          cargo doc "${cargo_args[@]}" --lib --no-deps --document-private-items

  release-validation:
    if: github.event_name == 'pull_request' && needs.validate-and-detect.outputs.has_rust_release_validation_tags == 'true'
    needs: validate-and-detect
    strategy:
      fail-fast: false
      matrix:
        tag: ${{ fromJson(needs.validate-and-detect.outputs.rust_release_validation_tags) }}
    permissions:
      contents: read
      pull-requests: read
      id-token: write
    uses: ./.github/workflows/release-rust-python-package.yaml
    with:
      tag: ${{ matrix.tag }}
      repository: testpypi
      publish_enabled: false

  create-release-tags:
    needs:
      - validate-and-detect
      - build-test
      - security-policy
      - mutation-testing
      - coverage
      - documentation
      - release-validation
    if: >-
      ${{
        always() &&
        github.event_name == 'push' &&
        github.ref == 'refs/heads/main' &&
        needs.validate-and-detect.outputs.has_rust_release_validation_tags == 'true' &&
        needs.build-test.result == 'success' &&
        needs.security-policy.result == 'success' &&
        needs.coverage.result == 'success' &&
        needs.documentation.result == 'success' &&
        (needs.mutation-testing.result == 'success' || needs.mutation-testing.result == 'skipped') &&
        (needs.release-validation.result == 'success' || needs.release-validation.result == 'skipped')
      }}
    runs-on: ubuntu-latest
    permissions:
      actions: write
      contents: write
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0

      - name: Create release tags for bumped plugin versions
        env:
          GH_TOKEN: ${{ github.token }}
          RELEASE_TAGS: ${{ needs.validate-and-detect.outputs.rust_release_validation_tags }}
        run: |
          set -euo pipefail
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          while IFS= read -r tag; do
            if [[ ! "${tag}" =~ ^[a-zA-Z0-9._-]+$ ]]; then
              echo "ERROR: unexpected characters in tag '${tag}'; aborting." >&2
              exit 1
            fi
            if remote_sha="$(git ls-remote --exit-code --tags origin "refs/tags/${tag}" | awk '{print $1}')"; then
              if [[ "${remote_sha}" != "${GITHUB_SHA}" ]]; then
                echo "FATAL: tag '${tag}' already exists at ${remote_sha}, expected ${GITHUB_SHA}." >&2
                exit 1
              fi
              echo "Tag ${tag} already exists remotely at ${GITHUB_SHA}; skipping tag creation."
            else
              if git show-ref --verify --quiet "refs/tags/${tag}"; then
                local_sha="$(git rev-list -n 1 "${tag}")"
                if [[ "${local_sha}" != "${GITHUB_SHA}" ]]; then
                  echo "FATAL: local tag '${tag}' exists at ${local_sha}, expected ${GITHUB_SHA}." >&2
                  exit 1
                fi
              else
                git tag "${tag}" "${GITHUB_SHA}"
              fi
              git push origin "refs/tags/${tag}" || {
                if remote_sha="$(git ls-remote --exit-code --tags origin "refs/tags/${tag}" | awk '{print $1}')" &&
                  [[ "${remote_sha}" == "${GITHUB_SHA}" ]]; then
                  echo "Tag ${tag} already exists remotely at ${GITHUB_SHA}; skipping."
                else
                  echo "FATAL: push of '${tag}' failed." >&2
                  exit 1
                fi
              }
            fi
            gh workflow run release-rust-python-package.yaml \
              --ref "${tag}" \
              -f "tag=${tag}" \
              -f repository=pypi \
              -f publish_enabled=true
            release_run_url="$(gh run list -w release-rust-python-package.yaml --limit 1 --json url --jq '.[0].url')"
            echo "Dispatched PyPI release workflow for ${tag}: ${release_run_url}"
          done < <(python3 -c 'import json, os; [print(tag) for tag in json.loads(os.environ["RELEASE_TAGS"])]')