python.yml

name: python

# The Python module `coreav` (docs/design/python.md): abi3 wheels built with
# maturin, tested with pytest against the generated fixtures. Wheels are
# kept as artifacts; nothing is published.
#
# What runs when, to keep Actions minutes down (billed minutes per run, each
# job rounded up: fixtures 2, Linux x86-64 3, Linux arm64 3 to 4, macOS arm64
# 1 x 10):
# - pushes to main and ready pull requests that change what is packaged (the
#   npm packages, the Python module, the command, the lock file, the policy
#   examples the tests run over, or this file): fixtures and Linux x86-64
#   only (5 minutes). Narrowed from the whole of `crates/**` while this
#   repository is private and its Actions minutes are billed; the weekly run
#   still builds every wheel;
# - tags, manual runs, a weekly run, and pull requests labelled `wheels`:
#   every wheel, Linux arm64 and macOS arm64 included (18 to 19 minutes).
#   The label is read from the pull request, so any event of a labelled
#   pull request builds them, not only the labelling itself.
# Draft pull requests skip it.
#
# The filter sits on the jobs (`python-changes`), not on the trigger: a
# workflow a trigger filter skips reports no check at all, and
# `python-report` has to appear on every pull request for the branch ruleset
# to require it (docs/design/ci-checks.md).
on:
  push:
    branches: [main]
    tags: ['v*']
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review, labeled]
  # A group of the merge queue, filtered by what the group changes on main
  # (docs/design/ci-checks.md, "The merge queue").
  merge_group:
    types: [checks_requested]
  schedule:
    # Every wheel once a week (Monday 04:00 UTC), on main.
    - cron: '0 4 * * 1'
  workflow_dispatch:

permissions:
  contents: read
  actions: read

jobs:
  python-changes:
    concurrency:
      group: ${{ github.workflow }}-python-changes-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
      cancel-in-progress: ${{ github.event_name == 'pull_request' }}
    # Whether this event's changed files need the jobs below at all. A
    # manual run, and anything whose changed files cannot be read, runs
    # them (docs/design/ci-checks.md).
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    outputs:
      run: ${{ steps.result.outputs.reuse != 'true' && steps.filter.outputs.run == 'true' }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          persist-credentials: false
          # A pull request's merge commit and its first parent, the base.
          fetch-depth: 2
      - id: result
        uses: ./.github/actions/merge-group-result
        with:
          workflow: python.yml
      - id: filter
        uses: ./.github/actions/changed-paths
        with:
          paths: |
            packages/cli/**
            crates/coreav-py/**
            crates/coreav-cli/**
            Cargo.lock
            # The policy examples both test suites run over.
            docs/schema/policy-examples/**
            scripts/oracle-ffmpeg.env
            scripts/install-ci-oracle-ffmpeg.sh
            .github/actions/oracle-cache/action.yml
            scripts/maturin.env
            scripts/install-ci-maturin.sh
            scripts/fetch-ci-archive.sh
            scripts/tests/test_maturin_downloads.py
            .github/actions/ci-archive-cache/**
            .github/workflows/python.yml
            .github/actions/changed-paths/**
            .github/actions/merge-group-result/**
            scripts/merge_group_result.py

  python-fixtures:
    concurrency:
      group: ${{ github.workflow }}-python-fixtures-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
      cancel-in-progress: ${{ github.event_name == 'pull_request' }}
    # Draft pull requests skip it. A label event does the same work as any
    # other pull-request event, `wheels` or not: a label must never report
    # a green `python-report` over the failure an earlier event of the same
    # commit found (docs/design/ci-checks.md, "What a report job must not
    # hide"). Labels are rare, and the filter above still decides.
    needs: python-changes
    if: >-
      needs.python-changes.outputs.run == 'true' &&
      (github.event_name != 'pull_request' || !github.event.pull_request.draft)
    runs-on: ubuntu-24.04
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          persist-credentials: false
      - uses: ./.github/actions/oracle-cache
      - name: Install the pinned FFmpeg
        run: scripts/install-ci-oracle-ffmpeg.sh "$RUNNER_TEMP/coreav-oracle-bin"
      - name: Generate the fixtures
        env:
          COREAV_REQUIRE_ALL_FIXTURES: '1'
        run: |
          python3 -I scripts/oracle_pin.py --verify-bin "$RUNNER_TEMP/coreav-oracle-bin"
          echo "$RUNNER_TEMP/coreav-oracle-bin" >> "$GITHUB_PATH"
          export PATH="$RUNNER_TEMP/coreav-oracle-bin:$PATH"
          scripts/gen-fixtures.sh "$RUNNER_TEMP/coreav-fixtures"
      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: fixtures
          path: ${{ runner.temp }}/coreav-fixtures
          retention-days: 1

  linux-x86_64:
    concurrency:
      group: ${{ github.workflow }}-linux-x86_64-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
      cancel-in-progress: ${{ github.event_name == 'pull_request' }}
    needs: python-fixtures
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          persist-credentials: false
      - id: maturin-pin
        run: scripts/install-ci-maturin.sh metadata linux-x86_64 >> "$GITHUB_OUTPUT"
      - uses: ./.github/actions/ci-archive-cache
        with:
          tool: ${{ steps.maturin-pin.outputs.tool }}
          version: ${{ steps.maturin-pin.outputs.version }}
          sha256: ${{ steps.maturin-pin.outputs.sha256 }}
      - name: Build the wheel (manylinux)
        run: scripts/install-ci-maturin.sh build linux-x86_64 --profile dist --locked --out dist --manifest-path crates/coreav-py/Cargo.toml
      - name: Keep CI build glue out of the wheel
        run: scripts/install-ci-maturin.sh verify-artifacts linux-x86_64
      - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable
        with:
          toolchain: stable
      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}
      # The command the tests compare every result with.
      - name: Build the coreav binary
        run: cargo build --locked --release -p coreav-cli
      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
        with:
          name: fixtures
          path: ${{ runner.temp }}/coreav-fixtures
      # The oldest and the newest Python the abi3 wheel serves.
      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: |
            3.9
            3.13
      - name: Test the wheel against the coreav binary
        env:
          COREAV_FIXTURES: ${{ runner.temp }}/coreav-fixtures
          COREAV_CLI: ${{ github.workspace }}/target/release/coreav
          PYTHONDONTWRITEBYTECODE: '1'
        run: |
          set -euo pipefail
          python3 -I packages/cli/scripts/notices_check.py dist/*.whl
          for python in python3.9 python3.13; do
            "$python" -m venv "$RUNNER_TEMP/venv-$python"
            "$RUNNER_TEMP/venv-$python/bin/pip" install --quiet dist/*.whl pytest
            (cd "$RUNNER_TEMP" && "$RUNNER_TEMP/venv-$python/bin/python" -m pytest -q -p no:cacheprovider \
              "$GITHUB_WORKSPACE/crates/coreav-py/tests")
          done
      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: wheel-linux-x86_64
          path: dist/*.whl
          retention-days: 14

  linux-arm64:
    concurrency:
      group: ${{ github.workflow }}-linux-arm64-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
      cancel-in-progress: ${{ github.event_name == 'pull_request' }}
    # Every wheel: tags, manual and weekly runs, and the `wheels` label.
    if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || startsWith(github.ref, 'refs/tags/') || contains(github.event.pull_request.labels.*.name, 'wheels')
    needs: python-fixtures
    runs-on: ubuntu-24.04
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          persist-credentials: false
      # Cross-compiled in the manylinux image, so no arm64 runner is needed;
      # the test runs under QEMU.
      - id: maturin-pin
        run: scripts/install-ci-maturin.sh metadata linux-arm64 >> "$GITHUB_OUTPUT"
      - uses: ./.github/actions/ci-archive-cache
        with:
          tool: ${{ steps.maturin-pin.outputs.tool }}
          version: ${{ steps.maturin-pin.outputs.version }}
          sha256: ${{ steps.maturin-pin.outputs.sha256 }}
      - name: Build the wheel (manylinux, cross-compiled)
        run: scripts/install-ci-maturin.sh build linux-arm64 --profile dist --locked --out dist --manifest-path crates/coreav-py/Cargo.toml
      - name: Keep CI build glue out of the wheel
        run: scripts/install-ci-maturin.sh verify-artifacts linux-arm64
      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
        with:
          name: fixtures
          path: ${{ runner.temp }}/coreav-fixtures
      - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0
        with:
          platforms: arm64
      - name: Test the wheel on arm64 (emulated)
        run: |
          docker run --rm --platform linux/arm64 \
            -v "$GITHUB_WORKSPACE:/src:ro" -v "$RUNNER_TEMP/coreav-fixtures:/fixtures:ro" \
            -e COREAV_FIXTURES=/fixtures -e PYTHONDONTWRITEBYTECODE=1 \
            python:3.12-slim@sha256:16bf2b5c59a08523519d3c8589deca849285dee1d62a3653e124c3dba340f15e \
            sh -euc 'pip install --quiet /src/dist/*.whl pytest && cd /tmp && python -m pytest -q -p no:cacheprovider /src/crates/coreav-py/tests'
      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: wheel-linux-arm64
          path: dist/*.whl
          retention-days: 14

  macos-arm64:
    concurrency:
      group: ${{ github.workflow }}-macos-arm64-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
      cancel-in-progress: ${{ github.event_name == 'pull_request' }}
    # Every wheel: tags, manual and weekly runs, and the `wheels` label.
    # Built natively: a wheel cross-compiled on Linux with zig records a
    # minimum of macOS 15 whatever deployment target is asked for, so it
    # cannot serve macOS 11 to 14 (docs/design/python.md).
    if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || startsWith(github.ref, 'refs/tags/') || contains(github.event.pull_request.labels.*.name, 'wheels')
    needs: python-fixtures
    runs-on: macos-15
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          persist-credentials: false
      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: '3.12'
      - id: maturin-pin
        run: scripts/install-ci-maturin.sh metadata macos-arm64 >> "$GITHUB_OUTPUT"
      - uses: ./.github/actions/ci-archive-cache
        with:
          tool: ${{ steps.maturin-pin.outputs.tool }}
          version: ${{ steps.maturin-pin.outputs.version }}
          sha256: ${{ steps.maturin-pin.outputs.sha256 }}
      - name: Build the wheel
        run: scripts/install-ci-maturin.sh build macos-arm64 --profile dist --locked --out dist --manifest-path crates/coreav-py/Cargo.toml
      - name: Keep CI build glue out of the wheel
        run: scripts/install-ci-maturin.sh verify-artifacts macos-arm64
      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
        with:
          name: fixtures
          path: ${{ runner.temp }}/coreav-fixtures
      - name: Test the wheel
        env:
          COREAV_FIXTURES: ${{ runner.temp }}/coreav-fixtures
          PYTHONDONTWRITEBYTECODE: '1'
        run: |
          set -euo pipefail
          python -m venv "$RUNNER_TEMP/venv"
          "$RUNNER_TEMP/venv/bin/pip" install --quiet dist/*.whl pytest
          cd "$RUNNER_TEMP" && "$RUNNER_TEMP/venv/bin/python" -m pytest -q -p no:cacheprovider \
            "$GITHUB_WORKSPACE/crates/coreav-py/tests"
      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: wheel-macos-arm64
          path: dist/*.whl
          retention-days: 14

  python-report:
    # Always runs, including when the filter above skipped every job, so
    # that the branch ruleset has one check name this workflow reports on
    # every pull request, a documentation-only one included
    # (docs/design/ci-checks.md).
    # Not `linux-arm64` or `macos-arm64`: those run on the `wheels`
    # label, which belongs to the pull request and not to the commit, so
    # a required check that covered them would say different things about
    # the same commit before and after a label changed
    # (docs/design/ci-checks.md, "What a report job must not hide").
    needs: [python-changes, python-fixtures, linux-x86_64]
    if: always()
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    steps:
      - name: Fail if a job above failed or was cancelled
        env:
          RESULTS: ${{ toJSON(needs) }}
        run: |
          set -euo pipefail
          { echo '| job | result |'
            echo '|---|---|'
            jq -r 'to_entries[] | "| \(.key) | \(.value.result) |"' <<< "$RESULTS"
          } >> "$GITHUB_STEP_SUMMARY"
          bad=$(jq -r '[to_entries[]
            | select(.value.result == "failure" or .value.result == "cancelled")
            | .key] | join(", ")' <<< "$RESULTS")
          if [ -n "$bad" ]; then
            echo "::error::failed or cancelled: $bad"
            exit 1
          fi