From 8d1f48727185d543b0c9772adec9b187efffd28a Mon Sep 17 00:00:00 2001 From: "Harlow, Jordan" Date: Fri, 6 Nov 2026 10:15:08 -0700 Subject: [PATCH 1/2] Build and test conda packages and wheels Adds a ci/ directory and .github/workflows/intel-ci.yml. The workflow builds conda-recipe/ with conda-build and the wheels with cibuildwheel, for every platform and Python version the recipe and pyproject.toml declare, and tests each package in a clean environment with ci/test.py. conda-recipe/ is now the only recipe, the conda-forge one that was conda-recipe-cf/; the Intel-channel recipe and conda-package.yml, which copied wheels out through WHEELS_OUTPUT_FOLDER, are removed. conda-package-cf.yml reports the same check names, so the required checks still run. conda_build_config.yaml now lists the Python versions. The wheels link MKL's layers, the Intel OpenMP runtime and icx's runtime, so their run-time requirements on mkl, intel-openmp and intel-cmplr-lib-rt come from [tool.gh-ci] wheel-requires in pyproject.toml, which ci/build.py adds to each wheel: in [project] they would fail the recipe's pip check, as conda-forge's mkl has no pip metadata. The wheels build against mkl-devel >=2026.2,<2027, the range they require at run time, and the Linux wheels are tagged manylinux_2_28 only, like the mkl wheel. ci/build.py keeps only the $ORIGIN entries of each ELF file's RPATH and RUNPATH in a wheel, as meson-python keeps the isolated build env's absolute lib directory, and refuses a file that would be left with none. ci/hooks.py tests each wheel in a venv made from conda-forge's Python, as pip users install it. Every python: entry in conda_build_config.yaml names its ABI, which ci/matrix.py requires, and every push and tag run gets a concurrency group of its own. --- .github/AGENTS.md | 4 +- .github/workflows/conda-package-cf.yml | 4 +- .github/workflows/conda-package.yml | 473 ----------------- .github/workflows/coverity.yml | 7 +- .github/workflows/intel-ci.yml | 188 +++++++ AGENTS.md | 5 +- README.md | 2 +- ci/README.md | 33 ++ ci/benchmark.py | 84 +++ ci/build.py | 428 +++++++++++++++ ci/config.yml | 13 + ci/hooks.py | 452 ++++++++++++++++ ci/install_icx.py | 62 +++ ci/matrix.py | 679 ++++++++++++++++++++++++ ci/test.py | 51 ++ conda-recipe-cf/AGENTS.md | 21 - conda-recipe-cf/bld.bat | 10 - conda-recipe-cf/build.sh | 8 - conda-recipe-cf/conda_build_config.yaml | 22 - conda-recipe-cf/meta.yaml | 60 --- conda-recipe/AGENTS.md | 35 +- conda-recipe/bld.bat | 21 +- conda-recipe/build.sh | 41 +- conda-recipe/conda_build_config.yaml | 15 +- conda-recipe/meta.yaml | 29 +- meson.build | 9 +- mkl_umath/tests/AGENTS.md | 2 +- pyproject.toml | 32 +- 28 files changed, 2081 insertions(+), 709 deletions(-) delete mode 100644 .github/workflows/conda-package.yml create mode 100644 .github/workflows/intel-ci.yml create mode 100644 ci/README.md create mode 100644 ci/benchmark.py create mode 100644 ci/build.py create mode 100644 ci/config.yml create mode 100644 ci/hooks.py create mode 100644 ci/install_icx.py create mode 100644 ci/matrix.py create mode 100644 ci/test.py delete mode 100644 conda-recipe-cf/AGENTS.md delete mode 100644 conda-recipe-cf/bld.bat delete mode 100644 conda-recipe-cf/build.sh delete mode 100644 conda-recipe-cf/conda_build_config.yaml delete mode 100644 conda-recipe-cf/meta.yaml diff --git a/.github/AGENTS.md b/.github/AGENTS.md index 47576d02..e7d81242 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -3,8 +3,8 @@ CI/CD workflows and repo automation. ## Workflows (source of truth) -- `conda-package.yml` — Intel channel conda build/test pipeline - `conda-package-cf.yml` — conda-forge-oriented build/test pipeline +- `intel-ci.yml` — Intel CI: builds and tests the conda packages of `conda-recipe` and the wheels, for the platforms and Python versions the recipe and `pyproject.toml` declare (`ci/matrix.py`); see `ci/README.md` - `build-with-clang.yml` — Intel clang compatibility checks - `build-with-standard-clang.yml` — standard clang compatibility checks - `build_pip.yml` — pip build pipeline with pre-release NumPy @@ -12,5 +12,5 @@ CI/CD workflows and repo automation. - `openssf-scorecard.yml` — security scanning ## Policy -- Treat workflow YAML as canonical for platform/Python matrices. +- Treat workflow YAML as canonical for platform/Python matrices, except `intel-ci.yml`, whose conda matrix is `conda-recipe/conda_build_config.yaml` and whose wheel matrix is the Python classifiers in `pyproject.toml`. - Avoid doc claims about CI coverage unless present in workflow config. diff --git a/.github/workflows/conda-package-cf.yml b/.github/workflows/conda-package-cf.yml index 174f9d43..b3f66b54 100644 --- a/.github/workflows/conda-package-cf.yml +++ b/.github/workflows/conda-package-cf.yml @@ -98,7 +98,7 @@ jobs: --error-overlinking \ "${VERSIONS[@]}" \ "${CHANNELS[@]}" \ - conda-recipe-cf + conda-recipe - name: Upload artifact uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -285,7 +285,7 @@ jobs: - name: Build conda package run: | conda activate - conda build --no-test --error-overlinking --python "${{ matrix.python_spec || matrix.python }}" --numpy ${{ matrix.numpy }} -c conda-forge --override-channels conda-recipe-cf + conda build --no-test --error-overlinking --python "${{ matrix.python_spec || matrix.python }}" --numpy ${{ matrix.numpy }} -c conda-forge --override-channels conda-recipe - name: Upload artifact uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml deleted file mode 100644 index 0c3e50af..00000000 --- a/.github/workflows/conda-package.yml +++ /dev/null @@ -1,473 +0,0 @@ -name: Conda package - -on: - push: - branches: - - main - pull_request: - -permissions: - contents: read - -env: - PACKAGE_NAME: mkl_umath - MODULE_NAME: mkl_umath - TEST_ENV_NAME: test_mkl_umath - VER_SCRIPT1: "import json; f = open('ver.json', 'r'); j = json.load(f); f.close(); d = j['mkl_umath'][0];" - VER_SCRIPT2: "print('='.join((d[s] for s in ('version', 'build'))))" - CONDA_BUILD_VERSION: 26.3.0 - -jobs: - build_linux: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python: "3.10" - numpy: "2.2" - - python: "3.11" - numpy: "2.3" - - python: "3.12" - numpy: "2.3" - - python: "3.13" - numpy: "2.3" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - - python: "3.15" - numpy: "2.5" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - activate-environment: "" - pkgs-dirs: ~/.conda/pkgs - - - name: Update conda - run: | - conda update -n base --all - - - name: Install conda-build - run: conda install -n base conda-build=${{ env.CONDA_BUILD_VERSION }} -c conda-forge --override-channels - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Store conda paths as envs - run: | - echo "CONDA_BLD=${CONDA}/conda-bld/linux-64" >> "$GITHUB_ENV" - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE/" >> "$GITHUB_ENV" - - - name: Build conda package - run: | - CHANNELS=(-c "https://software.repos.intel.com/python/conda" -c "conda-forge" --override-channels) - VERSIONS=(--python "${{ matrix.python_spec || matrix.python }}" --numpy "${{ matrix.numpy }}") - TEST=(--no-test) - - conda build \ - "${TEST[@]}" \ - "${VERSIONS[@]}" \ - "${CHANNELS[@]}" \ - conda-recipe - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.CONDA_BLD }}/${{ env.PACKAGE_NAME }}-*.conda - - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - test_linux: - needs: build_linux - runs-on: ${{ matrix.runner }} - - strategy: - matrix: - python: ["3.10", "3.11", "3.12", "3.13"] - numpy: ['numpy">=2"'] - experimental: [false] - runner: [ubuntu-latest] - include: - - python: "3.14" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - - python: "3.14" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - - python: "3.15" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - numpy: 'numpy">=2"' - experimental: false - runner: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} - env: - CHANNELS: -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels - - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - activate-environment: "" - pkgs-dirs: ~/.conda/pkgs - - name: Update conda - run: | - conda update -n base --all - - name: Install conda-index - run: | - conda install -n base conda-index -c conda-forge --override-channels - - name: Show Conda info - run: | - conda info --all - - name: List base environment packages - run: | - conda list -n base - - name: Create conda channel - run: | - mkdir -p "$GITHUB_WORKSPACE/channel/linux-64" - mv "${PACKAGE_NAME}"-*.conda "$GITHUB_WORKSPACE/channel/linux-64" - conda index "$GITHUB_WORKSPACE/channel" - # Test channel - conda search "$PACKAGE_NAME" -c "$GITHUB_WORKSPACE/channel" --override-channels - - name: Test conda channel - run: | - conda search "$PACKAGE_NAME" -c "$GITHUB_WORKSPACE"/channel --override-channels --info --json > "$GITHUB_WORKSPACE"/ver.json - cat "$GITHUB_WORKSPACE"/ver.json - - name: Collect dependencies - run: | - CHANNELS=(-c "$GITHUB_WORKSPACE/channel" -c "https://software.repos.intel.com/python/conda" -c "conda-forge" --override-channels) - PACKAGE_VERSION="$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}")" - conda create -n "${{ env.TEST_ENV_NAME }}" "${PACKAGE_NAME}=${PACKAGE_VERSION}" "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} "${CHANNELS[@]}" --only-deps --dry-run > lockfile - - name: Display lockfile - run: cat lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Install mkl_umath - run: | - CHANNELS=(-c "$GITHUB_WORKSPACE/channel" -c "https://software.repos.intel.com/python/conda" -c "conda-forge" --override-channels) - PACKAGE_VERSION="$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}")" - conda create -n "${{ env.TEST_ENV_NAME }}" "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} "$PACKAGE_NAME=${PACKAGE_VERSION}" pytest "${CHANNELS[@]}" - # Test installed packages - conda list -n "${{ env.TEST_ENV_NAME }}" - - - name: Smoke test - run: | - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.TEST_ENV_NAME }}" - python -c "import mkl_umath, numpy as np; mkl_umath.patch_numpy_umath(); np.sin(np.linspace(0, 1, num=10**6));" - - - name: Run tests - run: | - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.TEST_ENV_NAME }}" - pytest -v --pyargs ${{ env.MODULE_NAME }} - - build_windows: - runs-on: windows-latest - - strategy: - matrix: - include: - - python: "3.10" - numpy: "2.2" - - python: "3.11" - numpy: "2.3" - - python: "3.12" - numpy: "2.3" - - python: "3.13" - numpy: "2.3" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - - python: "3.14" - numpy: "2.3" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - - python: "3.15" - numpy: "2.5" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - env: - conda-bld: C:\Miniconda\conda-bld\win-64\ - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - activate-environment: build - channels: conda-forge - python-version: ${{ matrix.python }} - conda-build-version: ${{ env.CONDA_BUILD_VERSION }} - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Store conda paths as envs - shell: bash -l {0} - run: | - echo "CONDA_BLD=$CONDA/conda-bld/win-64/" | tr "\\\\" '/' >> "$GITHUB_ENV" - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE\\" >> "$GITHUB_ENV" - - - name: Install conda build - run: | - conda activate - conda install -y conda-build - conda list -n base - - - name: Build conda package - run: | - conda activate - conda build --no-test --python "${{ matrix.python_spec || matrix.python }}" --numpy ${{ matrix.numpy }} -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels conda-recipe - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.CONDA_BLD }}${{ env.PACKAGE_NAME }}-*.conda - - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python_tag || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - test_windows: - needs: build_windows - runs-on: ${{ matrix.runner }} - defaults: - run: - shell: cmd /C CALL {0} - strategy: - matrix: - python: ["3.10", "3.11", "3.12", "3.13"] - numpy: ['numpy">=2"'] - experimental: [false] - runner: [windows-latest] - include: - - python: "3.14" - python_spec: "3.14.* *_cp314" - python_tag: "3.14" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - - python: "3.14" - python_spec: "3.14.* *_cp314t" - python_tag: "3.14t" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - - python: "3.15" - python_spec: "3.15.* *_cp315" - python_tag: "3.15" - numpy: 'numpy">=2"' - experimental: false - runner: windows-latest - continue-on-error: ${{ matrix.experimental }} - env: - workdir: '${{ github.workspace }}' - CHANNELS: -c https://software.repos.intel.com/python/conda -c conda-forge --override-channels - - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python_tag || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - activate-environment: ${{ env.TEST_ENV_NAME }} - channels: conda-forge - conda-remove-defaults: "true" - python-version: ${{ matrix.python }} - - - name: Install conda-index - run: | - conda install -n base conda-index - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Create conda channel with the artifact bit - shell: cmd /C CALL {0} - run: | - echo ${{ env.workdir }} - mkdir ${{ env.workdir }}\channel - mkdir ${{ env.workdir }}\channel\win-64 - move ${{ env.PACKAGE_NAME }}-*.conda ${{ env.workdir }}\channel\win-64 - dir ${{ env.workdir }}\channel\win-64 - - - name: Index the channel - shell: cmd /C CALL {0} - run: | - conda index ${{ env.workdir }}\channel - - - name: Dump mkl_umath version info from created channel to STDOUT - shell: cmd /C CALL {0} - run: | - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.workdir }}/channel --override-channels --info --json - - - name: Dump mkl_umath version info from created channel into ver.json - shell: cmd /C CALL {0} - run: | - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.workdir }}/channel --override-channels --info --json > ${{ env.workdir }}\ver.json - - - name: Output content of workdir - shell: pwsh - run: Get-ChildItem -Path ${{ env.workdir }} - - - name: Output content of produced ver.json - shell: pwsh - run: Get-Content -Path ${{ env.workdir }}\ver.json - - - name: Collect dependencies - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - SET "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} --only-deps --dry-run > lockfile - - - name: Display lockfile content - shell: pwsh - run: Get-Content -Path .\lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 0 # Increase to reset cache - with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python_tag || matrix.python }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - # add intel-openmp as an explicit dependency - # to avoid it being missed when package version is specified exactly - - name: Install mkl_umath - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - set "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - SET "TEST_DEPENDENCIES=pytest pytest-cov" - SET "WORKAROUND_DEPENDENCIES=intel-openmp" - SET "DEPENDENCIES=%TEST_DEPENDENCIES% %WORKAROUND_DEPENDENCIES%" - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% %DEPENDENCIES% "python=${{ matrix.python_spec || matrix.python }}" ${{ matrix.numpy }} -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} - - - name: Report content of test environment - shell: cmd /C CALL {0} - run: | - CALL conda activate - echo "Value of CONDA environment variable was: " %CONDA% - echo "Value of CONDA_PREFIX environment variable was: " %CONDA_PREFIX% - CALL conda info && CALL conda list -n ${{ env.TEST_ENV_NAME }} - - - name: Smoke test - shell: cmd /C CALL {0} - run: | - @ECHO ON - CALL conda activate ${{ env.TEST_ENV_NAME }} - python -c "import mkl_umath, numpy as np; mkl_umath.patch_numpy_umath(); np.sin(np.linspace(0, 1, num=10**6));" - - - name: Run tests - shell: cmd /C CALL {0} - run: | - CALL conda activate ${{ env.TEST_ENV_NAME }} - python -m pytest -v -s --pyargs ${{ env.MODULE_NAME }} diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index c71de1d1..3ad30da4 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -79,11 +79,8 @@ jobs: run: | # shellcheck disable=SC1091 source "${ONEAPI_ROOT}/setvars.sh" - # Build with icx to match the shipped conda packages (both - # conda-recipe and conda-recipe-cf use CC=icx). meson.build only - # applies -fveclib=SVML, -fvectorize, -fimf-precision=high and - # -fprotect-parens when the compiler id is Intel, so gcc would not - # exercise that branch. + # Build with icx, as conda-recipe does: meson.build adds its SVML + # and precision flags only for an Intel compiler. export CC="${CMPLR_ROOT}/bin/icx" # icx is not auto-configured by the Coverity front-end, so register it # explicitly first. Two constraints on this command: diff --git a/.github/workflows/intel-ci.yml b/.github/workflows/intel-ci.yml new file mode 100644 index 00000000..81964b60 --- /dev/null +++ b/.github/workflows/intel-ci.yml @@ -0,0 +1,188 @@ +# Builds and tests mkl_umath's conda packages and wheels, see ci/README.md +name: Intel CI + +on: + push: + branches: [main] + tags: ['*'] + pull_request: + +permissions: + contents: read + +# A push or tag run gets a group of its own, so a queued main push is never dropped. +concurrency: + group: intel-ci-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + matrix: + name: Platforms and Python versions + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 10 + outputs: + matrix: ${{ steps.declared.outputs.matrix }} + wheel: ${{ steps.declared.outputs.wheel }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Read the declared platforms and Python versions + id: declared + run: | + python ci/matrix.py --event "$GITHUB_EVENT_NAME" --github-output matrix + python ci/matrix.py --system wheel --event "$GITHUB_EVENT_NAME" --github-output wheel + + # macOS runs on Intel runners, as MKL has no Apple silicon build. + build: + name: ${{ matrix.name }} Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The recipe takes its version from git describe. + fetch-depth: 0 + + # Miniforge 26.9.4-0 fails setup-miniconda's conda init on Windows runners; 26.7.2-0 works. + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install conda-build + run: conda install -y -q -n base --override-channels -c conda-forge conda-build + + - name: Build and run the recipe tests + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mkl_umath ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.conda + if-no-files-found: error + # Pull requests keep them 3 days, tags 30. + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test: + name: Test ${{ matrix.name }} Python ${{ matrix.python }} + needs: [matrix, build] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: mkl_umath ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the package and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" + + build-wheel: + name: ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install cibuildwheel + run: python -m pip install --no-input cibuildwheel==4.3.5 + + - name: Build the wheel + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --system wheel --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mkl_umath ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.whl + if-no-files-found: error + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test-wheel: + name: Test ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: [matrix, build-wheel] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: mkl_umath ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the wheel and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --kind wheel --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" diff --git a/AGENTS.md b/AGENTS.md index 265f9110..0b0fea0a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,8 +43,8 @@ CC=icx pip install --no-deps --no-build-isolation . # clang is also supported i - **Platforms:** Linux, Windows - **Python versions:** 3.10, 3.11, 3.12, 3.13, 3.14, 3.15 - **Workflows:** `.github/workflows/` - - `conda-package.yml` — main conda build/test pipeline - `conda-package-cf.yml` — conda-forge-oriented build/test pipeline + - `intel-ci.yml` — Intel CI: builds and tests the conda packages and the wheels; see `ci/README.md` - `build_pip.yml` — validates pip build with pre-release NumPy - `build-with-clang.yml` — Intel clang compatibility check - `build-with-standard-clang.yml` — standard clang compatibility check @@ -97,8 +97,7 @@ Below directories have local `AGENTS.md` for deeper context: - `mkl_umath/AGENTS.md` — Python API and code generation - `mkl_umath/src/AGENTS.md` — C/Cython implementation layer - `mkl_umath/tests/AGENTS.md` — unit tests and validation -- `conda-recipe/AGENTS.md` — Intel channel conda packaging -- `conda-recipe-cf/AGENTS.md` — conda-forge compatible recipe +- `conda-recipe/AGENTS.md` — the conda recipe - `_vendored/AGENTS.md` — vendored NumPy utilities --- diff --git a/README.md b/README.md index 9e6cafad..a186832e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Conda package](https://github.com/IntelPython/mkl_umath/actions/workflows/conda-package.yml/badge.svg)](https://github.com/IntelPython/mkl_umath/actions/workflows/conda-package.yml) +[![Conda package using conda-forge](https://github.com/IntelPython/mkl_umath/actions/workflows/conda-package-cf.yml/badge.svg)](https://github.com/IntelPython/mkl_umath/actions/workflows/conda-package-cf.yml) [![Build using pip and pre-release NumPy](https://github.com/IntelPython/mkl_umath/actions/workflows/build_pip.yml/badge.svg)](https://github.com/IntelPython/mkl_umath/actions/workflows/build_pip.yml) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/IntelPython/mkl_umath/badge)](https://securityscorecards.dev/viewer/?uri=github.com/IntelPython/mkl_umath) [![Coverity Scan Build Status](https://scan.coverity.com/projects/33201/badge.svg)](https://scan.coverity.com/projects/intelpython-mkl_umath) diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000..67ea2451 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,33 @@ +# How Intel CI tests mkl_umath + +`.github/workflows/intel-ci.yml` builds mkl_umath's conda packages and wheels in GitHub Actions and tests each one. It needs this directory, the recipe, and `pyproject.toml`. + +| File | Role | +|---|---| +| `../conda-recipe/` | The conda recipe. `conda_build_config.yaml` lists the Python versions. The recipe builds on Linux, Windows and Intel macOS unless a platform-only skip in `meta.yaml`, such as `skip: true # [win]`, rules one out. | +| `../pyproject.toml` | What the wheels are built from. Its Python classifiers list the wheels' Python versions, and its Operating System classifiers their platforms. `[tool.cibuildwheel]` holds the wheel build settings, and `[tool.gh-ci]` holds the run-time requirements the wheels need on top of `[project]`'s, which the conda packages get from the recipe. | +| `../.github/workflows/intel-ci.yml` | One job reads both matrices. One job per platform and Python version builds and uploads `mkl_umath Python `, and one builds `mkl_umath Wheels Python `. Then one job per package tests it. | +| `matrix.py` | Reads the platforms and Python versions from the recipe, or with `--system wheel` from `pyproject.toml`. A pull request gets the oldest and the newest regular versions. | +| `build.py` | Builds one Python version: with conda-build, which runs the recipe's own tests, or with `--system wheel` with cibuildwheel, in a manylinux container on Linux. The ELF files in a wheel keep only their `$ORIGIN` RPATH and RUNPATH entries. | +| `hooks.py` | The workflow's test jobs: installs one package into a clean environment with the requirements in `config.yml`, and runs `test.py`. A conda package's environment comes from conda-forge; a wheel goes into a venv made from conda-forge's Python, with everything else from PyPI. | +| `install_icx.py` | Installs Intel's icx, which the conda recipe builds with too, for the Windows wheel builds: `[tool.cibuildwheel.windows]` in `../pyproject.toml` runs it first. The Linux wheel builds get icx from Intel's yum repository. | +| `config.yml` | Names the workflow that builds the packages, its artifact names, and what the tests and benchmarks need. | +| `test.py` | Runs the installed package's test suite with pytest. | +| `benchmark.py` | Runs the asv suite in `../benchmarks/` against the installed package. | + +The scripts need only the Python standard library. The recipe skips macOS, as `conda-forge/mkl_umath-feedstock` does. + +**A pull request into `main`.** Actions builds the oldest and the newest regular Python versions on Linux and Windows, as a conda package and, where `pyproject.toml` declares the platform, as a wheel, and tests each package. + +**A push to `main` or a tag.** Actions builds and tests every declared version. + +To support another Python, add it to `conda-recipe/conda_build_config.yaml` for the conda packages and as a classifier in `pyproject.toml` for the wheels; nothing else lists versions. Try it locally from a conda base environment with conda-build, or with cibuildwheel for wheels: + +```bash +python ci/matrix.py # every platform and Python version +python ci/matrix.py --event pull_request # what a pull request builds +python ci/build.py --python VERSION --output /tmp/out # VERSION: one matrix.py lists +python ci/hooks.py --python VERSION --artifacts /tmp/out --work /tmp/hooks +python ci/build.py --system wheel --python VERSION --output /tmp/wheels +python ci/hooks.py --kind wheel --python VERSION --artifacts /tmp/wheels --work /tmp/wheel-hooks +``` diff --git a/ci/benchmark.py b/ci/benchmark.py new file mode 100644 index 00000000..6cb356bb --- /dev/null +++ b/ci/benchmark.py @@ -0,0 +1,84 @@ +""" +Benchmarks the installed package. + +Runs the asv suite in benchmarks/ against the package installed in this +environment, from an empty working directory, without a build of its own, and +records the results under the commit being tested, in $GH_CI_RESULTS. A pull +request samples each benchmark once, enough to show that every one still runs; +a tag measures them properly. A benchmark that fails fails the run; a slower one +does not, as nothing compares the timings with a baseline yet. +""" + +import json +import os +import pathlib +import subprocess +import sys + +# asv's own dependency, which reads the comments in its configuration. +import json5 + +ASV_CONFIG = pathlib.Path("benchmarks", "asv.conf.json") + + +def asv_config(source: pathlib.Path, results: pathlib.Path) -> pathlib.Path: + """Writes this run's asv configuration into the working directory. + + It is the repository's own, with absolute paths, since asv reads them from + where the configuration is, and with results under GH_CI_RESULTS. It lists + one branch, HEAD: the checkout is the commit under test, detached, and asv + cannot resolve the repository's branches there. + """ + path = source / ASV_CONFIG + config = json5.loads(path.read_text(encoding="utf-8")) + work = pathlib.Path.cwd() + config.update( + repo=str((path.parent / config.get("repo", "..")).resolve()), + benchmark_dir=str( + (path.parent / config.get("benchmark_dir", "benchmarks")).resolve() + ), + branches=["HEAD"], + results_dir=str(results / "asv"), + html_dir=str(work / "html"), + env_dir=str(work / "env"), + ) + written = work / "asv.conf.json" + written.write_text(json.dumps(config, indent=2), encoding="utf-8") + return written + + +def main() -> int: + """Runs the asv suite against the installed package.""" + config = asv_config( + pathlib.Path(os.environ["GH_CI_SOURCE"]), + pathlib.Path(os.environ["GH_CI_RESULTS"]), + ) + asv = [sys.executable, "-m", "asv"] + # asv keeps its machine description in ~/.asv-machine.json; this one stays + # with the run rather than with the machine's user. + home = pathlib.Path.cwd() / "home" + home.mkdir(exist_ok=True) + env = dict(os.environ, HOME=str(home)) + described = subprocess.run( + asv + ["machine", "--yes", "--config", str(config)], env=env + ) + if described.returncode: + return described.returncode + run = [ + "run", + "--config", + str(config), + f"--environment=existing:{sys.executable}", + f"--set-commit-hash={os.environ['GH_CI_COMMIT']}", + "--no-pull", + "--show-stderr", + "--launch-method=spawn", + ] + if os.environ.get("GH_CI_EVENT") != "tag": + run.append("--quick") + # asv exits 2 when a benchmark failed. + return subprocess.run(asv + run, env=env).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/build.py b/ci/build.py new file mode 100644 index 00000000..a2ee8bf5 --- /dev/null +++ b/ci/build.py @@ -0,0 +1,428 @@ +""" +Builds this repository's packages for one Python version. + +For the intel-ci.yml GitHub Actions workflow. + +This file and ci/matrix.py work out the build system from the files present, +and the Python versions from that build system's own declaration: + + rattler-build .conda recipe.yaml or recipe/recipe.yaml, versions in + variants.yaml + conda-build .conda conda-recipe/meta.yaml or meta.yaml, versions in + conda_build_config.yaml + cibuildwheel .whl pyproject.toml, versions from its classifiers + +Usage: + python ci/build.py --python 3.14 --output OUT [--system rattler-build] + [--channel conda-forge] + +--python must be one of the declared versions; `python ci/matrix.py` lists them. + +Packages land directly in OUT, ready for actions/upload-artifact. A conda +recipe's own tests run during the build. Needs only the Python standard library. +rattler-build is used from PATH, or downloaded at a pinned version for Linux or +Windows and checked against its published sha256; conda-build, or cibuildwheel +for wheels, must already be installed for this interpreter, so run it with the +conda base install's Python to build with conda-build. + +Wheels build with cibuildwheel, in a manylinux container on Linux, as +pyproject.toml's [tool.cibuildwheel] says. Run-time requirements that +pyproject.toml cannot declare, because the conda package gets them from its +recipe instead, such as MKL's libraries, go in [tool.gh-ci] wheel-requires: +build.py adds them to each wheel's metadata. It also cuts the RPATH and +RUNPATH of each ELF file in a wheel to their $ORIGIN entries, dropping absolute +ones such as the isolated build environment's lib directory, which meson-python +keeps, and refuses a file that would have none left. +""" + +import argparse +import base64 +import hashlib +import os +import platform +import shlex +import shutil +import struct +import subprocess +import sys +import tempfile +import urllib.request +import zipfile +from pathlib import Path + +import matrix + +ROOT = matrix.ROOT +PACKAGE_SUFFIXES = (".conda", ".tar.bz2", ".whl") +# cibuildwheel's platform part of a build identifier, per system. +WHEEL_PLATFORMS = { + "Linux": "manylinux_x86_64", + "Windows": "win_amd64", + "Darwin": "macosx_x86_64", +} +# The ELF program header and dynamic section values the RUNPATH check reads. +PT_LOAD, PT_DYNAMIC = 1, 2 +DT_NULL, DT_STRTAB, DT_RPATH, DT_RUNPATH = 0, 5, 15, 29 +ORIGIN_PREFIXES = ("$ORIGIN", "${ORIGIN}") + +# The rattler-build release fetched when none is on PATH, per system, with the +# file name it goes under and the sha256 its release publishes. +RATTLER_BUILD_VERSION = "v0.77.0" +RATTLER_BUILD_RELEASES = { + "Linux": ( + "rattler-build-x86_64-unknown-linux-musl", + "rattler-build", + "5a5e8150c703b21e88a01f9130ea49a7a680198e9ac9414170c0a293fed7e5af", + ), + "Windows": ( + "rattler-build-x86_64-pc-windows-msvc.exe", + "rattler-build.exe", + "e144334f782cee9aa23e26eb8d095c143e1696fbbec7ba7534a156663d4abb2c", + ), +} + + +class BuildError(Exception): + """Raised when the build cannot start or produced nothing.""" + + +def ensure_rattler_build(tools: Path) -> None: + """ + Puts rattler-build on PATH, downloading the pinned release when none is + there. + """ + if shutil.which("rattler-build"): + return + release = RATTLER_BUILD_RELEASES.get(platform.system()) + if release is None or platform.machine().lower() not in ("x86_64", "amd64"): + raise BuildError( + "no rattler-build on PATH, and only the Linux and Windows x86_64 " + "releases are pinned here" + ) + asset, name, sha256 = release + print(f"build: fetching rattler-build {RATTLER_BUILD_VERSION}", flush=True) + url = ( + "https://github.com/prefix-dev/rattler-build/releases/download/" + f"{RATTLER_BUILD_VERSION}/{asset}" + ) + with urllib.request.urlopen(url, timeout=300) as reply: + binary = reply.read() + if hashlib.sha256(binary).hexdigest() != sha256: + raise BuildError( + f"rattler-build {RATTLER_BUILD_VERSION} does not match its pinned " + "sha256" + ) + tools.mkdir(parents=True, exist_ok=True) + target = tools / name + target.write_bytes(binary) + target.chmod(0o755) + os.environ["PATH"] = f'{tools}{os.pathsep}{os.environ.get("PATH", "")}' + + +def cibuildwheel_identifier(python: matrix.Python) -> str: + """The one build cibuildwheel runs, such as cp314t-manylinux_x86_64.""" + tag = WHEEL_PLATFORMS.get(platform.system()) + if tag is None: + raise BuildError( + f"wheels build on Linux, Windows or macOS, not {platform.system()}" + ) + minor, threaded = matrix.version_key(python.version) + return f'cp3{minor}{"t" if threaded else ""}-{tag}' + + +def wheel_requires() -> list[str]: + """[tool.gh-ci] wheel-requires from pyproject.toml.""" + if matrix.TOMLLIB is None: + raise BuildError("reading pyproject.toml needs Python 3.11 or newer") + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + tool = matrix.TOMLLIB.loads(text).get("tool", {}).get("gh-ci", {}) + requires = tool.get("wheel-requires", []) + if not isinstance(requires, list) or not all( + isinstance(r, str) for r in requires + ): + raise BuildError("[tool.gh-ci] wheel-requires must be a list of specs") + return requires + + +def record_hash(data: bytes) -> str: + """A file's hash as a wheel's RECORD spells it.""" + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()) + return "sha256=" + digest.rstrip(b"=").decode("ascii") + + +def elf_search_paths(data: bytes, name: str) -> list[tuple[int, int, str]]: + """ + The RPATH and RUNPATH strings of an ELF file, as (file offset, length, + value); none for a file that is not ELF. + """ + if data[:4] != b"\x7fELF": + return [] + if data[4:6] != b"\x02\x01": + raise BuildError(f"{name}: only 64-bit little-endian ELF is read") + (phoff,) = struct.unpack_from(" tuple[bytes, list[str]]: + """ + An ELF file with its RPATH and RUNPATH cut to their $ORIGIN entries, in + place as patchelf does, and the entries it dropped. + """ + edited, dropped = bytearray(data), [] + for start, length, value in elf_search_paths(data, name): + paths = value.split(":") + kept = [path for path in paths if path.startswith(ORIGIN_PREFIXES)] + if kept == paths: + continue + # An empty RUNPATH would mean the working directory + if not kept: + raise BuildError(f"{name}: no $ORIGIN entry in RUNPATH {value!r}") + edited[start : start + length] = ( + ":".join(kept).encode().ljust(length, b"\0") + ) + dropped += [path for path in paths if path not in kept] + return bytes(edited), dropped + + +def with_requirements(metadata: bytes, requirements: list[str]) -> bytes: + """A wheel's METADATA with Requires-Dist lines added.""" + text = metadata.decode("utf-8") + # Some Windows build backends write the metadata with CRLF. + eol = "\r\n" if "\r\n" in text else "\n" + head, blank, body = text.partition(eol * 2) + head = head.rstrip("\r\n") + "".join( + f"{eol}Requires-Dist: {r}" for r in requirements + ) + return (head + (eol * 2 + body if blank else eol)).encode("utf-8") + + +def rewrite_wheel(wheel: Path, requirements: list[str]) -> list[str]: + """ + Adds Requires-Dist lines to a built wheel and cuts its ELF files' RPATH + and RUNPATH to their $ORIGIN entries, updating its RECORD; lists the + entries it dropped. + """ + with zipfile.ZipFile(wheel) as source: + members = [(info, source.read(info)) for info in source.infolist()] + names = [info.filename for info, _ in members] + metadata = next(n for n in names if n.endswith(".dist-info/METADATA")) + record = metadata.removesuffix("METADATA") + "RECORD" + changed, dropped = {}, [] + for index, (info, data) in enumerate(members): + if info.filename == metadata and requirements: + new = with_requirements(data, requirements) + else: + new, paths = origin_only(data, info.filename) + dropped += [f"{path} from {info.filename}" for path in paths] + if new != data: + members[index] = (info, new) + changed[info.filename] = new + if not changed: + return dropped + for index, (info, data) in enumerate(members): + if info.filename == record: + lines = [ + ( + f"{name},{record_hash(changed[name])},{len(changed[name])}" + if (name := line.split(",")[0]) in changed + else line + ) + for line in data.decode("utf-8").splitlines() + ] + members[index] = (info, ("\n".join(lines) + "\n").encode()) + rewritten = wheel.with_suffix(".tmp") + with zipfile.ZipFile(rewritten, "w", zipfile.ZIP_DEFLATED) as target: + for info, data in members: + target.writestr(info, data) + rewritten.replace(wheel) + return dropped + + +def commands( + system: str, + selector: Path, + python: matrix.Python, + channels: list[str], + scratch: Path, +) -> list[list[str]]: + """ + The commands that build one Python version's packages into scratch. + + Each build system gets the version back exactly as its own file declares it, + and keeps every other variant setting from that file. + """ + channel_args = [arg for channel in channels for arg in ("-c", channel)] + if system == "rattler-build": + return [ + [ + "rattler-build", + "build", + "-r", + str(selector), + "--variant", + f"python={python.spec}", + "--output-dir", + str(scratch / "conda"), + *channel_args, + ] + ] + if system == "conda-build": + # conda runs as a module of this interpreter, so Windows needs no + # conda.bat on PATH. + return [ + [ + sys.executable, + "-m", + "conda", + "build", + str(selector.parent), + "--python", + python.spec, + "--no-anaconda-upload", + "--output-folder", + str(scratch / "conda"), + "--override-channels", + *channel_args, + ] + ] + return [ + [ + sys.executable, + "-m", + "cibuildwheel", + "--only", + cibuildwheel_identifier(python), + "--output-dir", + str(scratch / "wheel"), + str(ROOT), + ] + ] + + +def collect(scratch: Path, output: Path) -> list[Path]: + """Moves built packages into output, flat, and lists the ones it moved.""" + output.mkdir(parents=True, exist_ok=True) + built = [] + for package in sorted(scratch.rglob("*")): + if ( + package.is_file() + and package.name.endswith(PACKAGE_SUFFIXES) + and "broken" not in package.parts + ): + built.append(output / package.name) + shutil.move(str(package), built[-1]) + if not built: + raise BuildError(f"the build left no package in {output}") + return built + + +def main(argv: list[str] | None = None) -> int: + """Builds one Python version.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--output", required=True, type=Path, help="where the packages go" + ) + parser.add_argument( + "--system", + default="", + choices=("", *matrix.SYSTEMS), + help="build system to use instead of the detected one", + ) + parser.add_argument( + "--channel", + action="append", + default=None, + help="conda channel to build against, repeatable; default conda-forge", + ) + default_tools = ( + Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) / "ci-tools" + ) + parser.add_argument( + "--tools", + type=Path, + default=default_tools, + help="where a downloaded rattler-build goes", + ) + args = parser.parse_args(argv) + + system, selector = matrix.detect(ROOT, args.system) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise BuildError( + f"Python {args.python} is not declared for {system}; " + f'declared: {", ".join(declared)}' + ) + python = declared[args.python] + print( + f"build: {system} from {selector.relative_to(ROOT)} " + f"for Python {python.version} ({python.spec})", + flush=True, + ) + if system == "rattler-build": + ensure_rattler_build(args.tools) + output = args.output.resolve() + with tempfile.TemporaryDirectory() as scratch: + for command in commands( + system, + selector, + python, + args.channel or ["conda-forge"], + Path(scratch), + ): + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True) + built = collect(Path(scratch), output) + if system == "wheel": + requires = wheel_requires() + for wheel in built: + for dropped in rewrite_wheel(wheel, requires): + print(f"build: dropped RUNPATH entry {dropped}") + if requires: + print("build: added Requires-Dist " + ", ".join(requires)) + print("build: " + ", ".join(path.name for path in built)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + BuildError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"build: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/config.yml b/ci/config.yml new file mode 100644 index 00000000..5f79cd54 --- /dev/null +++ b/ci/config.yml @@ -0,0 +1,13 @@ +# Intel CI settings for mkl_umath, see ci/README.md +version: 1 +workflow: intel-ci.yml +artifacts: + conda: "mkl_umath {platform} Python {python}" + wheel: "mkl_umath {platform} Wheels Python {python}" +test: + requires: [pytest] +benchmark: + # PyYAML, which asv needs, has no PyPI wheel for every Python the recipe declares. + requires: [psutil, pyyaml] + # conda-forge lacks asv for some Python the recipe declares; the abi3 wheel on PyPI fits all. + pip_requires: [asv] diff --git a/ci/hooks.py b/ci/hooks.py new file mode 100644 index 00000000..c6897e2f --- /dev/null +++ b/ci/hooks.py @@ -0,0 +1,452 @@ +""" +Runs ci/test.py against one built artifact. + +For the GitHub Actions test job. + +Each Python version's artifact is installed into a fresh conda environment from +a local channel plus conda-forge, pinned to its exact build, with the test +requirements ci/config.yml lists, and the test runs with that environment's +Python from an empty working directory, on Linux or Windows. A wheel goes into +a venv made from conda-forge's Python, with the test requirements, by pip, as +the wheel's users install it. The test gets the variables ci/test.py reads, and +nothing else from the runner. + +Usage, from any Python 3.10 or newer; it runs itself again with the conda base +install's Python, from $CONDA, which setup-miniconda sets, or the conda on PATH: + python ci/hooks.py --python 3.14 --artifacts DIR --work DIR [--kind wheel] + +Needs conda, conda-package-handling and ruamel.yaml, which every conda base has, +and installs nothing into it. +""" + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import matrix + +# Dependencies of conda itself; YAML is read with PyYAML, else ruamel.yaml +try: + from conda_package_handling import api as CPH_API +except ModuleNotFoundError: + CPH_API = None +try: + import yaml as PYYAML +except ModuleNotFoundError: + PYYAML = None +try: + from ruamel.yaml import YAML as RUAMEL_YAML +except ModuleNotFoundError: + RUAMEL_YAML = None + +ROOT = matrix.ROOT +IS_WINDOWS = os.name == "nt" +if IS_WINDOWS: + PLATFORM, HOST_SUBDIR = "windows", "win-64" +elif sys.platform == "darwin": + PLATFORM, HOST_SUBDIR = "macos", "osx-64" +else: + PLATFORM, HOST_SUBDIR = "linux", "linux-64" +CONDA_SUFFIXES = (".conda", ".tar.bz2") +# All the test sees of the runner's environment, so no tokens reach it +KEEP = ( + "PATH", + "HOME", + "USER", + "LANG", + "TERM", + "TMPDIR", + "TZ", + "http_proxy", + "https_proxy", + "no_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CONDA_PKGS_DIRS", + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "PATHEXT", + "TEMP", + "TMP", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "PROGRAMDATA", + "NUMBER_OF_PROCESSORS", + "PROCESSOR_ARCHITECTURE", +) + + +class HookError(Exception): + """Raised when the artifact cannot be installed or ci/test.py is missing.""" + + +def run(command: list[str], **options: object) -> None: + """Echoes and runs a command, never through a shell.""" + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True, **options) + + +def conda(*args: str) -> None: + """Runs conda from the base environment running this script.""" + run([sys.executable, "-m", "conda", *args]) + + +def empty_repodata(subdir: str) -> dict: + """The repodata.json of a channel subdir with no packages.""" + return { + "info": {"subdir": subdir}, + "packages": {}, + "packages.conda": {}, + "repodata_version": 1, + } + + +def package_record(package: Path) -> tuple[str, dict]: + """A conda package's subdir and repodata record, from its index.json.""" + if CPH_API is None: + raise HookError( + "run this with a conda base Python, " + "which has conda-package-handling" + ) + with tempfile.TemporaryDirectory() as scratch: + CPH_API.extract(str(package), scratch, components="info") + record = json.loads( + (Path(scratch) / "info" / "index.json").read_text(encoding="utf-8") + ) + data = package.read_bytes() + record.update( + { + "md5": hashlib.md5(data, usedforsecurity=False).hexdigest(), + "sha256": hashlib.sha256(data).hexdigest(), + "size": len(data), + } + ) + subdir = str(record.get("subdir") or "noarch") + if not re.fullmatch(r"[a-z0-9]+(-[a-z0-9_]+)?", subdir): + raise HookError( + f"{package.name} declares an unusable subdir {subdir!r}" + ) + return subdir, record + + +def stage_channel(packages: list[Path], channel: Path) -> None: + """ + Lays the packages out as a local channel, by the subdir each one declares, + and writes its repodata.json itself, without conda-index. + """ + repodata = { + subdir: empty_repodata(subdir) for subdir in ("noarch", HOST_SUBDIR) + } + for package in packages: + subdir, record = package_record(package) + (channel / subdir).mkdir(parents=True, exist_ok=True) + shutil.copy2(package, channel / subdir / package.name) + section = ( + "packages.conda" if package.name.endswith(".conda") else "packages" + ) + repodata.setdefault(subdir, empty_repodata(subdir))[section][ + package.name + ] = record + for subdir, data in repodata.items(): + (channel / subdir).mkdir(parents=True, exist_ok=True) + (channel / subdir / "repodata.json").write_text( + json.dumps(data, indent=2) + "\n", encoding="utf-8" + ) + + +def test_python(version: str) -> str: + """The conda spec for the test environment's interpreter.""" + minor, threaded = matrix.version_key(version) + # The .* matters: conda reads a bare 3.11 as exactly 3.11.0 + if minor < matrix.FREE_THREADED_MINOR: + return f"3.{minor}.*" + # conda-forge marks the ABI from 3.13, so 3.14 can't resolve to 3.14t + return f'3.{minor}.* *_cp3{minor}{"t" if threaded else ""}' + + +def test_requirements() -> tuple[list[str], list[str], list[str] | None]: + """ + The conda-forge and pip packages ci/config.yml installs next to the + artifact, and a wheel's PyPI requirements, None when it gives none. + """ + text = (ROOT / "ci" / "config.yml").read_text(encoding="utf-8") + if PYYAML is not None: + config = PYYAML.safe_load(text) + elif RUAMEL_YAML is not None: + config = RUAMEL_YAML(typ="safe", pure=True).load(text) + else: + raise HookError("reading ci/config.yml needs PyYAML or ruamel.yaml") + test = (config or {}).get("test") or {} + lists = [] + for key in ("requires", "pip_requires", "wheel_requires"): + values = test.get(key) + if values is not None and ( + not isinstance(values, list) + or not all(isinstance(v, str) for v in values) + ): + raise HookError( + f"ci/config.yml: test.{key} must be a list of package specs" + ) + lists.append(values) + return lists[0] or [], lists[1] or [], lists[2] + + +def exact_spec(package: Path) -> str: + """name=version=build from a conda package file name.""" + stem = package.name + for suffix in CONDA_SUFFIXES: + stem = stem.removesuffix(suffix) + name, version, build = stem.rsplit("-", 2) + return f"{name}={version}={build}" + + +def env_python(env_dir: Path, kind: str = "conda") -> Path: + """The interpreter of a conda environment, or of a wheel's venv.""" + if not IS_WINDOWS: + return env_dir / "bin" / "python" + if kind == "wheel": + return env_dir / "Scripts" / "python.exe" + return env_dir / "python.exe" + + +def env_path(env_dir: Path, kind: str) -> list[Path]: + """ + The directories activating the conda environment, or a wheel's venv, would + put first on PATH. + """ + if not IS_WINDOWS: + return [env_dir / "bin"] + if kind == "wheel": + return [env_dir / "Scripts"] + library = env_dir / "Library" + return [ + env_dir, + library / "mingw-w64" / "bin", + library / "usr" / "bin", + library / "bin", + env_dir / "Scripts", + env_dir / "bin", + ] + + +def test_env( + env_dir: Path, artifacts: Path, results: Path, args: argparse.Namespace +) -> dict: + """The variables ci/test.py reads, and no runner tokens.""" + env = {key: value for key, value in os.environ.items() if key in KEEP} + env["PATH"] = os.pathsep.join( + [*map(str, env_path(env_dir, args.kind)), env.get("PATH", "")] + ) + prefix = "VIRTUAL_ENV" if args.kind == "wheel" else "CONDA_PREFIX" + env.update( + { + prefix: str(env_dir), + "GH_CI_ARTIFACTS": str(artifacts), + "GH_CI_SOURCE": str(ROOT), + "GH_CI_RESULTS": str(results), + "GH_CI_KIND": args.kind, + "GH_CI_PYTHON": args.python, + "GH_CI_PLATFORM": PLATFORM, + "GH_CI_EVENT": args.event, + "GH_CI_COMMIT": args.commit, + } + ) + return env + + +def summarize(lines: list[str]) -> None: + """Adds the results to the job summary page, in GitHub Actions.""" + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as page: + page.write("\n".join(lines) + "\n") + + +def install(args: argparse.Namespace) -> Path: + """ + Creates the test environment with the artifact, pinned to its exact build; + returns it. + """ + system, selector = matrix.detect( + ROOT, "wheel" if args.kind == "wheel" else "" + ) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise HookError( + f"Python {args.python} is not declared; " + f'declared: {", ".join(declared)}' + ) + suffixes = (".whl",) if args.kind == "wheel" else CONDA_SUFFIXES + packages = sorted( + path + for path in args.artifacts.rglob("*") + if path.name.endswith(suffixes) + ) + if not packages: + raise HookError(f"{args.artifacts} holds no {args.kind} package") + requires, pip_requires, wheel_requires = test_requirements() + env_dir = args.work / "env" + pip = [ + str(env_python(env_dir, args.kind)), + "-m", + "pip", + "install", + "--no-input", + ] + python = f"python {test_python(args.python)}" + if args.kind == "wheel": + base = args.work / "python" + conda_create(base, python) + run([str(env_python(base)), "-m", "venv", str(env_dir)]) + wheel_specs = requires if wheel_requires is None else wheel_requires + run([*pip, *map(str, packages), *wheel_specs]) + else: + channel = args.work / "channel" + stage_channel(packages, channel) + conda_create( + env_dir, + python, + *requires, + *map(exact_spec, packages), + channels=(channel.as_uri(),), + ) + if pip_requires: + run([*pip, *pip_requires]) + return env_dir + + +def conda_create( + env_dir: Path, *specs: str, channels: tuple[str, ...] = () +) -> None: + """Creates a conda environment from conda-forge and channels.""" + conda( + "create", + "-y", + "-q", + "-p", + str(env_dir), + "--override-channels", + *(arg for channel in channels for arg in ("-c", channel)), + "-c", + "conda-forge", + *specs, + ) + + +def run_test(env_dir: Path, args: argparse.Namespace) -> int: + """Runs ci/test.py with the environment's Python; returns its status.""" + script = ROOT / "ci" / "test.py" + if not script.is_file(): + raise HookError("ci/test.py is required and missing") + results, scratch = ( + args.work / "results" / "test", + args.work / "scratch" / "test", + ) + for directory in (results, scratch): + directory.mkdir(parents=True, exist_ok=True) + command = [str(env_python(env_dir, args.kind)), str(script)] + print(f"+ {shlex.join(command)}", flush=True) + status = subprocess.run( + command, + cwd=scratch, + check=False, + env=test_env(env_dir, args.artifacts, results, args), + ).returncode + result = "passed" if status == 0 else f"failed, exit {status}" + summarize( + [ + f"### {matrix.PLATFORMS[PLATFORM]} Python {args.python}" + f" {args.kind}", + "", + "| Test | Result |", + "|---|---|", + f"| ci/test.py | {result} |", + ] + ) + return status + + +def conda_base_python() -> Path: + """The conda base install's Python: from $CONDA, else the conda on PATH.""" + root = os.environ.get("CONDA", "") + if not root: + found = shutil.which("conda") + if not found: + raise HookError( + "no conda base install: set CONDA or put conda on PATH" + ) + # conda lives in bin, Scripts or condabin, one level below the install. + root = str(Path(found).resolve().parent.parent) + python = env_python(Path(root)) + if not python.is_file(): + raise HookError( + f"{root} is not a conda base install; it has no {python.name}" + ) + return python + + +def main(argv: list[str] | None = None) -> int: + """Installs the artifact, then runs ci/test.py against it.""" + if CPH_API is None: + # Re-run with base's Python, which has conda-package-handling + command = [ + str(conda_base_python()), + str(Path(__file__).resolve()), + *(sys.argv[1:] if argv is None else argv), + ] + return subprocess.run(command, check=False).returncode + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--artifacts", required=True, type=Path, help="the downloaded artifact" + ) + parser.add_argument( + "--work", required=True, type=Path, help="an empty scratch directory" + ) + parser.add_argument( + "--event", default="push", choices=("pr", "tag", "push") + ) + parser.add_argument("--commit", default="") + parser.add_argument( + "--kind", + default="conda", + choices=("conda", "wheel"), + help="what the artifact holds", + ) + args = parser.parse_args(argv) + args.artifacts, args.work = args.artifacts.resolve(), args.work.resolve() + if args.work.exists() and ( + not args.work.is_dir() or any(args.work.iterdir()) + ): + raise HookError(f"--work {args.work} must be an empty directory") + return run_test(install(args), args) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + HookError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"hooks: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/install_icx.py b/ci/install_icx.py new file mode 100644 index 00000000..631eafbc --- /dev/null +++ b/ci/install_icx.py @@ -0,0 +1,62 @@ +""" +Installs Intel's icx for mkl_umath's Windows wheel builds. + +cibuildwheel runs this before the builds, as pyproject.toml says. It fetches +micromamba, checked against its published sha256, and creates PREFIX with +conda-forge's dpcpp_win-64, the compiler the conda recipe builds with. + +Usage: + python ci/install_icx.py PREFIX +""" + +import hashlib +import subprocess +import sys +import tempfile +import urllib.request +from pathlib import Path + +MICROMAMBA = ( + "https://github.com/mamba-org/micromamba-releases/releases/download/" + "2.10.0-0/micromamba-win-64" +) +MICROMAMBA_SHA256 = ( + "f8b4284bd4eb1134122eaf93b3ea6c7be567e596de5829e1a6f478e34e41974a" +) +COMPILER = "dpcpp_win-64=2026.2.0" + + +def main() -> int: + """Creates PREFIX with the compiler.""" + prefix = Path(sys.argv[1]) + with tempfile.TemporaryDirectory() as scratch: + micromamba = Path(scratch) / "micromamba.exe" + with urllib.request.urlopen(MICROMAMBA, timeout=300) as response: + micromamba.write_bytes(response.read()) + digest = hashlib.sha256(micromamba.read_bytes()).hexdigest() + if digest != MICROMAMBA_SHA256: + print( + f"install_icx: micromamba has sha256 {digest}, " + f"not {MICROMAMBA_SHA256}", + file=sys.stderr, + ) + return 1 + command = [ + str(micromamba), + "create", + "-y", + "-r", + str(Path(scratch) / "root"), + "-p", + str(prefix), + "--override-channels", + "-c", + "conda-forge", + COMPILER, + ] + print("+ " + " ".join(command), flush=True) + return subprocess.run(command, check=False).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/matrix.py b/ci/matrix.py new file mode 100644 index 00000000..5b197bd7 --- /dev/null +++ b/ci/matrix.py @@ -0,0 +1,679 @@ +""" +The platforms and Python versions this repository builds. + +Both are read from the repository's own build files. + +Every build system already has a standard place for both, so each is written +once, where the build reads it, and the GitHub Actions matrix follows it: + + rattler-build, selected by recipe.yaml + Python versions python: in variants.yaml next to the recipe + platforms build: skip: in the recipe + conda-build, selected by meta.yaml + Python versions python: in conda_build_config.yaml next to the recipe + platforms skip: true # [selector] lines in the recipe + wheel, selected by pyproject.toml + Python versions the Programming Language :: Python :: 3.N classifiers; + any Free Threading classifier adds 3.Nt for each from 3.14 + platforms the Operating System :: classifiers + +A recipe builds on every platform in PLATFORMS, Linux, Windows and macOS on +Intel, unless a skip that depends on the platform alone rules one out, such as +`skip: [win]` or `skip: true # [osx]`; `osx and arm64` rules out none, as +macOS here is Intel. A skip that also depends on something else, such as +`win and py < 311`, or a skip: line that cannot be read, is refused, as the +matrix would still list the builds it skips. A pyproject.toml without Operating +System classifiers, or with OS Independent, builds on all three. Of the +declared platforms, the matrix holds those GitHub Actions has runners for, +BUILT_PLATFORMS. + +meson-python, scikit-build-core and every other PEP 517 backend declare these in +pyproject.toml too, so the wheel row covers them. Another convention, such as +pixi.toml, is one more reader in READERS and PLATFORM_READERS and, if it is a +new build system, one more entry in SYSTEMS. + +Usage: + python ci/matrix.py [--system SYSTEM] [--event EVENT] [--github-output NAME] + +Prints the build matrix as JSON for strategy.matrix: {"include": [{"platform": +"linux", "name": "Linux", "python": "3.14"}, ...]}, every platform with every +Python version. For a pull_request event only the oldest and the newest regular +versions are listed. --github-output also writes NAME= to +$GITHUB_OUTPUT, for later jobs. Standard library only; reading pyproject.toml +needs Python 3.11 or newer for tomllib. +""" + +import argparse +import ast +import json +import os +import re +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +# Only the wheel reader needs it, and only it needs Python 3.11 or newer. +try: + import tomllib as TOMLLIB +except ModuleNotFoundError: + TOMLLIB = None + +ROOT = Path(__file__).resolve().parent.parent + +# Build system -> files that select it, in the order they are looked for. +SYSTEMS = { + "rattler-build": ("recipe.yaml", "recipe/recipe.yaml"), + "conda-build": ( + "meta.yaml", + "conda-recipe/meta.yaml", + "recipe/meta.yaml", + ), + "wheel": ("pyproject.toml",), +} + +# The platforms CI builds and tests, and how artifact names spell them. +PLATFORMS = {"linux": "Linux", "windows": "Windows", "macos": "macOS"} +# The platforms the workflow can build on: GitHub-hosted runners have all +# three. A declared platform missing here is reported and left out. +BUILT_PLATFORMS = ("linux", "windows", "macos") +# What a platform-only selector means on each platform, for both conda-build and +# rattler-build. +SELECTORS = { + "linux": { + "linux": True, + "linux64": True, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + "windows": { + "linux": False, + "linux64": False, + "unix": False, + "win": True, + "win64": True, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + # Intel Macs only, as MKL has no Apple silicon build. + "macos": { + "linux": False, + "linux64": False, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": True, + "osx64": True, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, +} +CONDA_SKIP_RE = re.compile(r"\s*skip\s*:") +# conda-build's own selector pattern, which lets a comment follow the selector +CONDA_SELECTOR_RE = re.compile(r"(.+?)\s*(#.*)?\[([^\[\]]+)\](?(2)[^\(\)]*)$") +OS_CLASSIFIERS = { + "Operating System :: OS Independent": tuple(PLATFORMS), + "Operating System :: POSIX": ("linux",), + "Operating System :: Unix": ("linux",), + "Operating System :: POSIX :: Linux": ("linux",), + "Operating System :: Microsoft :: Windows": ("windows",), + "Operating System :: MacOS": ("macos",), +} + +PULL_REQUEST_EVENTS = ("pull_request",) +OLDEST_MINOR = 10 +FREE_THREADED_MINOR = 13 +# A Free Threading classifier adds 3.Nt from 3.14, the first supported one, +# which is also the oldest cibuildwheel builds. +CLASSIFIED_FREE_THREADED_MINOR = 14 + +# A conda python variant with its ABI: 3.12.* *_cpython, 3.13.* *_cp313 or +# 3.14.* *_cp314t; from 3.13 it must be cp3N or cp3Nt. +CONDA_SPEC_RE = re.compile(r"3\.(\d+)\.\*\s+\*_(?:cpython|cp3(\d+)(t?))") +CLASSIFIER_RE = re.compile(r"Programming Language :: Python :: 3\.(\d+)") +FREE_THREADING_CLASSIFIER = "Programming Language :: Python :: Free Threading" +LOWER_BOUND_RE = re.compile(r">=\s*3\.(\d+)") + + +class DeclarationError(Exception): + """ + Raised when the build files declare no Python versions, or ones this cannot + read. + """ + + +@dataclass(frozen=True) +class Python: + """One declared Python version.""" + + version: str # as artifact names spell it: 3.14 or 3.14t + spec: str # as the build system spells it, passed back to it unchanged + + +def version_key(version: str) -> tuple[int, bool]: + """(minor, free-threaded); also the sort order.""" + return int(version.rstrip("t").split(".")[1]), version.endswith("t") + + +def checked(minor: int, threaded: bool, where: str) -> str: + """The version string, once it is one CI can test.""" + if minor < OLDEST_MINOR or (threaded and minor < FREE_THREADED_MINOR): + raise DeclarationError( + f'{where}: 3.{minor}{"t" if threaded else ""} is not supported; ' + f"the oldest is 3.{OLDEST_MINOR}, free-threaded from " + f"3.{FREE_THREADED_MINOR}" + ) + return f'3.{minor}{"t" if threaded else ""}' + + +def _strip_comment(line: str, where: str) -> str: + """ + The line without a trailing comment; conda-build selectors are refused, not + guessed at. + """ + in_quote = "" + for index, char in enumerate(line): + if char in "\"'" and in_quote in ("", char): + in_quote = "" if in_quote else char + elif ( + char == "#" + and not in_quote + and (index == 0 or line[index - 1].isspace()) + ): + if line[index:].lstrip("# ").startswith("["): + raise DeclarationError( + f"{where}: selectors in the python list are not supported" + ) + return line[:index].rstrip() + return line.rstrip() + + +def _unquote(value: str, where: str) -> str: + """A plain or quoted YAML scalar.""" + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + return value[1:-1] + if not value or value[0] in "{[&*!|>" or ":" in value: + raise DeclarationError( + f"{where}: write each python entry as a plain value, not {value!r}" + ) + return value + + +def yaml_python_list(text: str, where: str) -> list[str]: + """ + The top-level python: list of a variants file, without a YAML library. + + Accepts a block list or a one-line [a, b] list of plain or quoted values. + Anything richer, such as if/then selectors or templating, is refused so the + matrix is never guessed. + """ + lines = text.splitlines() + for number, line in enumerate(lines): + if not re.match(r"python\s*:", line): + continue + rest = _strip_comment(line.split(":", 1)[1], where).strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + raise DeclarationError(f"{where}: python must be a list") + values = [] + for item in lines[number + 1 :]: + # Checked before the comment, as the next key may have a selector + if item[:1] not in ("", " ", "\t", "-", "#"): + break # the next top-level key + entry = _strip_comment(item, where).lstrip() + if not entry: + continue + if not entry.startswith("- "): + raise DeclarationError( + f"{where}: python must be a plain list of versions" + ) + values.append(_unquote(entry[2:], where)) + return values + raise DeclarationError(f"{where} has no top-level python: list") + + +def from_conda_variants(path: Path) -> list[Python]: + """The python: list of a variants.yaml or a conda_build_config.yaml.""" + where = ( + path.relative_to(ROOT).as_posix() + if path.is_relative_to(ROOT) + else str(path) + ) + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as ex: + raise DeclarationError( + f"{where} is missing; list the Python versions to build there" + ) from ex + pythons = [] + for spec in yaml_python_list(text, where): + match = CONDA_SPEC_RE.fullmatch(spec) + if ( + not match + or (match[2] and match[2] != match[1]) + or (not match[2] and int(match[1]) >= FREE_THREADED_MINOR) + ): + raise DeclarationError( + f"{where}: {spec!r} is not a CPython variant such as " + '"3.12.* *_cpython", "3.13.* *_cp313" or "3.14.* *_cp314t"' + ) + pythons.append( + Python(checked(int(match[1]), bool(match[3]), where), spec) + ) + return pythons + + +def from_pyproject(path: Path) -> list[Python]: + """ + The 3.N classifiers in pyproject.toml, plus 3.Nt from 3.14 if free-threading + is declared. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + project = TOMLLIB.loads(path.read_text(encoding="utf-8")).get("project", {}) + classifiers = project.get("classifiers", []) + minors = sorted( + { + int(match[1]) + for classifier in classifiers + if (match := CLASSIFIER_RE.fullmatch(classifier)) + } + ) + if not minors: + raise DeclarationError( + "pyproject.toml lists no Programming Language :: Python :: 3.N " + "classifiers" + ) + bound = LOWER_BOUND_RE.search(project.get("requires-python", "")) + if bound and minors[0] < int(bound[1]): + raise DeclarationError( + f"pyproject.toml classifies 3.{minors[0]}, below requires-python " + f'{project["requires-python"]}' + ) + threaded = any( + classifier.startswith(FREE_THREADING_CLASSIFIER) + for classifier in classifiers + ) + versions = [checked(minor, False, "pyproject.toml") for minor in minors] + versions += [ + f"3.{minor}t" + for minor in minors + if threaded and minor >= CLASSIFIED_FREE_THREADED_MINOR + ] + return [Python(version, version) for version in versions] + + +def skips_platform(expression: str, platform: str) -> bool | None: + """ + Whether a skip expression rules out a whole platform: True or False when it + depends on the platform alone, None when it also depends on anything else, + such as the Python version. + """ + try: + tree = ast.parse( + expression.strip().removeprefix("${{").removesuffix("}}").strip(), + mode="eval", + ).body + except SyntaxError: + return None + + def value(node: ast.AST) -> bool: + if isinstance(node, ast.BoolOp): + values = [value(item) for item in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not value(node.operand) + if isinstance(node, ast.Name) and node.id in SELECTORS[platform]: + return SELECTORS[platform][node.id] + if isinstance(node, ast.Constant) and isinstance(node.value, bool): + return node.value + raise ValueError(ast.dump(node)) + + try: + return value(tree) + except ValueError: + return None + + +def platforms_without(skips: list[str], where: str) -> list[str]: + """ + Every platform in PLATFORMS, but those a skip rules out; a skip that + depends on more than the platform is refused. + """ + for skip in skips: + if any( + skips_platform(skip, platform) is None for platform in PLATFORMS + ): + raise DeclarationError( + f"{where}: the skip {skip!r} depends on more than the " + "platform, so the matrix would still list the builds it skips" + ) + kept = [ + platform + for platform in PLATFORMS + if not any(skips_platform(skip, platform) for skip in skips) + ] + if not kept: + raise DeclarationError( + f"{where} skips every platform CI builds: " + + ", ".join(PLATFORMS.values()) + ) + return kept + + +def yaml_build_skips(text: str, where: str) -> list[str]: + """ + The skip: entries directly under a rattler-build recipe's top-level build:, + as a list of plain or quoted values, a one-line [a, b] list, or one value; + none when there is no skip. + """ + lines = text.splitlines() + starts = [ + number + for number, line in enumerate(lines) + if re.match(r"build\s*:\s*(#.*)?$", line) + ] + if not starts: + return [] + child = "" + for number in range(starts[0] + 1, len(lines)): + line = ( + _strip_comment(lines[number], where) + if "#" in lines[number] + else lines[number] + ) + if not line.strip(): + continue + if not line[0].isspace(): + return [] # the next top-level key + indent = line[: len(line) - len(line.lstrip())] + child = child or indent + match = ( + re.match(r"skip\s*:(.*)", line.lstrip()) + if indent == child + else None + ) + if not match: + continue + rest = match[1].strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + return [_unquote(rest, where)] + values = [] + for item in lines[number + 1 :]: + entry = item.strip() + if not entry or entry.startswith("#"): + continue + if not entry.startswith("- "): + break + values.append(_unquote(entry[2:].split(" #")[0], where)) + return values + return [] + + +def from_rattler_recipe(recipe: Path) -> list[str]: + """The platforms a rattler-build recipe builds on, from its build: skip:.""" + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + return platforms_without( + yaml_build_skips(recipe.read_text(encoding="utf-8"), where), where + ) + + +def from_conda_recipe(recipe: Path) -> list[str]: + """ + The platforms a conda-build meta.yaml builds on, from its skip: true # + [selector] lines; a skip: line it cannot read is refused. + """ + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + skips = [] + for line in recipe.read_text(encoding="utf-8").splitlines(): + if not CONDA_SKIP_RE.match(line): + continue + selected = CONDA_SELECTOR_RE.match(line) + key, selector = ( + (selected[1], selected[3]) + if selected + else (line.split(" #")[0], "True") + ) + value = key.split(":", 1)[1].strip() + if value in ("false", "False", "no"): + continue + if value not in ("true", "True", "yes"): + raise DeclarationError( + f"{where}: cannot read {line.strip()!r}; write " + "skip: true # [selector]" + ) + skips.append(selector) + return platforms_without(skips, where) + + +def from_os_classifiers(path: Path) -> list[str]: + """ + The platforms pyproject.toml's Operating System classifiers name; all of + them when it names none. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + classifiers = ( + TOMLLIB.loads(path.read_text(encoding="utf-8")) + .get("project", {}) + .get("classifiers", []) + ) + named = [ + classifier + for classifier in classifiers + if classifier.startswith("Operating System ::") + ] + if not named: + return list(PLATFORMS) + found = { + platform + for classifier in named + for prefix, platforms in OS_CLASSIFIERS.items() + if classifier == prefix or classifier.startswith(prefix + " ::") + for platform in platforms + } + if not found: + raise DeclarationError( + "pyproject.toml names no Operating System CI builds: " + + ", ".join(PLATFORMS.values()) + ) + return [platform for platform in PLATFORMS if platform in found] + + +# Build system -> where it declares Python versions, given the file that +# selected it. +READERS: dict[str, Callable[[Path], list[Python]]] = { + "rattler-build": lambda recipe: from_conda_variants( + recipe.parent / "variants.yaml" + ), + "conda-build": lambda recipe: from_conda_variants( + recipe.parent / "conda_build_config.yaml" + ), + "wheel": from_pyproject, +} + +# Build system -> where it declares the platforms it builds on, given the file +# that selected it. +PLATFORM_READERS: dict[str, Callable[[Path], list[str]]] = { + "rattler-build": from_rattler_recipe, + "conda-build": from_conda_recipe, + "wheel": from_os_classifiers, +} + + +def detect(root: Path, system: str = "") -> tuple[str, Path]: + """The build system and the file that selects it.""" + for name, candidates in SYSTEMS.items(): + if system and name != system: + continue + for candidate in candidates: + if (root / candidate).is_file(): + return name, root / candidate + wanted = system or " or ".join(SYSTEMS) + raise DeclarationError( + f"no {wanted} input found; looked for " + + ", ".join(c for name, cs in SYSTEMS.items() for c in cs) + ) + + +def declared(system: str, selector: Path) -> list[Python]: + """The build system's Python versions, sorted, each listed once.""" + pythons = READERS[system](selector) + versions = [python.version for python in pythons] + if not pythons: + raise DeclarationError(f"{system} declares no Python versions") + if len(set(versions)) != len(versions): + raise DeclarationError( + f"{system} declares a Python version twice: {versions}" + ) + return sorted(pythons, key=lambda python: version_key(python.version)) + + +def declared_platforms(system: str, selector: Path) -> list[str]: + """The platforms the build system builds on, in PLATFORMS order.""" + return PLATFORM_READERS[system](selector) + + +def for_event(pythons: list[Python], event: str) -> list[Python]: + """All of them, or for a pull request the oldest and newest regular ones.""" + if event not in PULL_REQUEST_EVENTS: + return pythons + regular = [python for python in pythons if not python.version.endswith("t")] + regular = regular or pythons + return list(dict.fromkeys(regular[:1] + regular[-1:])) + + +def build_matrix(system: str, selector: Path, event: str) -> dict: + """ + Every declared platform Actions can build on, with every Python version the + event builds. + """ + pythons = for_event(declared(system, selector), event) + platforms = declared_platforms(system, selector) + waiting = [ + PLATFORMS[platform] + for platform in platforms + if platform not in BUILT_PLATFORMS + ] + if waiting: + print( + f'matrix: {", ".join(waiting)} declared but not built: ' + "no runners for it yet", + file=sys.stderr, + ) + built = [platform for platform in platforms if platform in BUILT_PLATFORMS] + if not built: + raise DeclarationError( + "the build files declare only platforms there are no runners " + "for yet: " + ", ".join(waiting) + ) + return { + "include": [ + { + "platform": platform, + "name": PLATFORMS[platform], + "python": python.version, + } + for platform in built + for python in pythons + ] + } + + +def main(argv: list[str] | None = None) -> int: + """Prints the matrix to build; writes it for GitHub Actions if asked.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--system", + default="", + choices=("", *SYSTEMS), + help="build system to read instead of the detected one", + ) + parser.add_argument( + "--event", default="", help="GitHub event name, such as pull_request" + ) + parser.add_argument( + "--github-output", + default="", + metavar="NAME", + help="also write NAME= to $GITHUB_OUTPUT", + ) + args = parser.parse_args(argv) + if args.github_output: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", args.github_output): + raise DeclarationError( + f"{args.github_output!r} is not an output name" + ) + if not os.environ.get("GITHUB_OUTPUT"): + raise DeclarationError( + "--github-output needs $GITHUB_OUTPUT, which Actions sets" + ) + system, selector = detect(ROOT, args.system) + matrix = build_matrix(system, selector, args.event) + cells = ", ".join( + f'{cell["name"]} {cell["python"]}' for cell in matrix["include"] + ) + print( + f"matrix: {system} from {selector.relative_to(ROOT)}: {cells}", + file=sys.stderr, + ) + text = json.dumps(matrix) + print(text) + if args.github_output: + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"{args.github_output}={text}\n") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except DeclarationError as error: + print(f"matrix: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/test.py b/ci/test.py new file mode 100644 index 00000000..628413c4 --- /dev/null +++ b/ci/test.py @@ -0,0 +1,51 @@ +""" +Tests mkl_umath's installed package. + +Runs with one built conda package or wheel installed in a clean environment, as +ci/hooks.py sets up, from an empty working directory, so the tests import the +installed mkl_umath rather than the source tree. The test requirements come from +ci/config.yml. Writes a JUnit report to $GH_CI_RESULTS. +""" + +import os +import pathlib +import sys +import sysconfig + +import pytest + +PACKAGE = "mkl_umath" + + +def running_python() -> str: + """This interpreter's version as artifact names spell it, such as 3.14t.""" + version = f"{sys.version_info.major}.{sys.version_info.minor}" + threaded = sysconfig.get_config_var("Py_GIL_DISABLED") + return version + ("t" if threaded else "") + + +def main() -> int: + """Runs the installed package's test suite.""" + expected = os.environ["GH_CI_PYTHON"] + if running_python() != expected: + print(f"running Python {running_python()}, expected {expected}") + return 1 + # The suite name says which platform and Python ran. + cell = f"{os.environ.get('GH_CI_PLATFORM', 'linux')}.py{expected}" + report = pathlib.Path(os.environ["GH_CI_RESULTS"]) / f"{PACKAGE}.xml" + return pytest.main( + [ + "-v", + "-p", + "no:cacheprovider", + f"--junitxml={report}", + "-o", + f"junit_suite_name={PACKAGE} {cell}", + "--pyargs", + PACKAGE, + ] + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conda-recipe-cf/AGENTS.md b/conda-recipe-cf/AGENTS.md deleted file mode 100644 index dcc7eef5..00000000 --- a/conda-recipe-cf/AGENTS.md +++ /dev/null @@ -1,21 +0,0 @@ -# AGENTS.md — conda-recipe-cf/ - -Conda-forge compatible build recipe (alternative to Intel channel recipe). - -## Difference from conda-recipe/ -- No `conda_build_config.yaml` (uses conda-forge defaults) -- May use different compiler toolchain -- For conda-forge feedstock integration (if upstreamed) - -## Files -- **meta.yaml** — conda-forge compatible metadata, test commands (`pytest --pyargs mkl_umath`) -- **build.sh** / **bld.bat** — platform build scripts - -## Status -- Not currently used in main CI workflows -- Maintained for potential conda-forge submission -- Use `conda-recipe/` for Intel channel builds - -## Notes -- If upstreaming to conda-forge, this recipe should be preferred -- Compiler requirements may differ (Clang/GCC vs Intel icx) diff --git a/conda-recipe-cf/bld.bat b/conda-recipe-cf/bld.bat deleted file mode 100644 index 534ca591..00000000 --- a/conda-recipe-cf/bld.bat +++ /dev/null @@ -1,10 +0,0 @@ -REM A workaround for activate-dpcpp.bat issue to be addressed in 2021.4 -set "LIB=%BUILD_PREFIX%\Library\lib;%BUILD_PREFIX%\compiler\lib;%LIB%" -set "INCLUDE=%BUILD_PREFIX%\include;%INCLUDE%" - -set "CC=icx" -set "CXX=icx" - -REM -Ccompile-args=-v makes ninja print full compiler commands (verbose build) -%PYTHON% -m pip install --no-build-isolation --no-deps -Ccompile-args=-v . -vv -if errorlevel 1 exit 1 diff --git a/conda-recipe-cf/build.sh b/conda-recipe-cf/build.sh deleted file mode 100644 index a90d19b9..00000000 --- a/conda-recipe-cf/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -ex - -# -Ccompile-args=-v makes ninja print full compiler commands (verbose build) -CC=icx CXX=icpx $PYTHON -m pip install --no-build-isolation --no-deps \ - -Csetup-args="-Dmkl_threading=gnu_thread" \ - -Ccompile-args=-v \ - . \ - -vv diff --git a/conda-recipe-cf/conda_build_config.yaml b/conda-recipe-cf/conda_build_config.yaml deleted file mode 100644 index 6e19b3ef..00000000 --- a/conda-recipe-cf/conda_build_config.yaml +++ /dev/null @@ -1,22 +0,0 @@ -numpy: # as in conda-forge's global pinning; CI overrides it with --numpy - - '2' -c_compiler: # [linux] - - gcc # [linux] -c_compiler_version: # [linux] - - '15' # [linux] -cxx_compiler: # [linux] - - gxx # [linux] -cxx_compiler_version: # [linux] - - '15' # [linux] -c_stdlib: # [linux] - - sysroot # [linux] -c_stdlib_version: # [linux] - - '2.28' # [linux] -c_stdlib: # [win] - - vs # [win] -cxx_compiler: # [win] - - vs2022 # [win] -c_compiler: # [win] - - vs2022 # [win] -CL: - - /fp:precise # [win] diff --git a/conda-recipe-cf/meta.yaml b/conda-recipe-cf/meta.yaml deleted file mode 100644 index 60d7c8da..00000000 --- a/conda-recipe-cf/meta.yaml +++ /dev/null @@ -1,60 +0,0 @@ -{% set version = "0.6.0" %} -{% set buildnumber = 0 %} - -package: - name: mkl_umath - version: {{ version }} - -source: - path: ../ - -build: - number: {{ buildnumber }} - ignore_run_exports_from: - - {{ compiler('dpcpp') }} # [linux] - ignore_run_exports: - - blas - missing_dso_whitelist: - - $RPATH/mkl_umath_loops.dll # [win] - -requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ stdlib('c') }} - - {{ compiler('dpcpp') }} >=2024.2 # [not osx] - host: - - meson-python >=0.16.0 - - cmake - - ninja - - git - - cython >=3.1.0 - - python - - mkl-devel - - intel-cmplr-lib-rt # [not osx] - - numpy - - llvm-openmp - - pip - run: - - python - # NumPy 1 vs 2 loops are chosen at compile time (USE_NUMPY_2), so a build - # against NumPy 2 must not run on NumPy 1.x; run_exports add the upper bound - - numpy >=2 - -test: - requires: - - pip - - pytest - commands: - - pip check - - pytest --pyargs mkl_umath - imports: - - mkl_umath - - mkl_umath._ufuncs - - mkl_umath._patch_numpy - -about: - home: http://github.com/IntelPython/mkl_umath - license: BSD-3-Clause - license_file: LICENSE.txt - summary: Universal functions for real and complex floating point arrays powered by Intel(R) Math Kernel Library Vector (Intel(R) MKL) and Intel(R) Short Vector Math Library (Intel(R) SVML) diff --git a/conda-recipe/AGENTS.md b/conda-recipe/AGENTS.md index e378ac6d..e12ec52d 100644 --- a/conda-recipe/AGENTS.md +++ b/conda-recipe/AGENTS.md @@ -1,30 +1,13 @@ # AGENTS.md — conda-recipe/ -Conda package build recipe for Intel channel distribution. +The conda recipe, built against conda-forge only. ## Files -- **meta.yaml** — package metadata, dependencies, build requirements, test commands (`pytest --pyargs mkl_umath`) -- **build.sh** — Linux build script -- **bld.bat** — Windows build script -- **conda_build_config.yaml** — build matrix (Python versions, numpy pins) - -## Build configuration -- **Channels:** `https://software.repos.intel.com/python/conda`, `conda-forge` -- **Python versions:** 3.10, 3.11, 3.12, 3.13, 3.14, 3.15 -- **Build system:** meson-python (via `python -m build`) -- **Compilers:** Intel C compiler (icx) -- **Dependencies:** mkl-devel, intel-openmp, dpcpp_{linux,win}-64, numpy - -## Build outputs -- Conda package: `mkl_umath--.conda` -- Platform-specific: `linux-64/`, `win-64/` - -## CI usage -- Built in `.github/workflows/conda-package.yml` -- Artifacts uploaded per Python version -- Test stage uses built artifacts from channel - -## Maintenance -- Keep `conda_build_config.yaml` in sync with CI matrix -- NumPy pin: must match Intel channel NumPy versions -- MKL version: track oneAPI releases +- **meta.yaml** — metadata, dependencies, test commands (`pytest --pyargs mkl_umath`) +- **build.sh** / **bld.bat** — platform build scripts +- **conda_build_config.yaml** — the Python versions Intel CI builds, NumPy 2 and the compilers; + its `python:` list is read by `ci/matrix.py`: one version per line, no selectors + +## CI +- Built and tested by `conda-package-cf.yml` and by Intel CI (`intel-ci.yml`) +- Wheels don't come from this recipe: `intel-ci.yml` builds them from `pyproject.toml` with `ci/build.py` diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index 58178a84..534ca591 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -5,21 +5,6 @@ set "INCLUDE=%BUILD_PREFIX%\include;%INCLUDE%" set "CC=icx" set "CXX=icx" -%PYTHON% -m build -w -n -x -if %ERRORLEVEL% neq 0 exit 1 - -for /f %%f in ('dir /b /S .\dist') do ( - %PYTHON% -m pip install %%f ^ - --no-build-isolation ^ - --no-deps ^ - --only-binary :all: ^ - --no-index ^ - --prefix %PREFIX% ^ - -vv - if %ERRORLEVEL% neq 0 exit 1 -) - -if NOT "%WHEELS_OUTPUT_FOLDER%"=="" ( - copy dist\mkl_umath*.whl %WHEELS_OUTPUT_FOLDER% - if %ERRORLEVEL% neq 0 exit 1 -) +REM -Ccompile-args=-v makes ninja print full compiler commands (verbose build) +%PYTHON% -m pip install --no-build-isolation --no-deps -Ccompile-args=-v . -vv +if errorlevel 1 exit 1 diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index 4d25b512..a90d19b9 100644 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -1,41 +1,8 @@ #!/bin/bash -ex -# This is necessary to help DPC++ find Intel libraries such as SVML, IRNG, etc in build prefix -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${BUILD_PREFIX}/lib" - -# Intel LLVM must cooperate with compiler and sysroot from conda -echo "--gcc-toolchain=${BUILD_PREFIX} --sysroot=${BUILD_PREFIX}/${HOST}/sysroot -target ${HOST}" > icx_for_conda.cfg -ICXCFG="$(pwd)/icx_for_conda.cfg" -export ICXCFG - -read -r GLIBC_MAJOR GLIBC_MINOR <<< "$(conda list '^sysroot_linux-64$' \ - | tail -n 1 | awk '{print $2}' | grep -oP '\d+' | head -n 2 | tr '\n' ' ')" - -if [ -d "build" ]; then - rm -rf build -fi - -export CC=icx -export CXX=icpx - -# -wnx flags mean: --wheel --no-isolation --skip-dependency-check # -Ccompile-args=-v makes ninja print full compiler commands (verbose build) -${PYTHON} -m build -w -n -x -Ccompile-args=-v - -${PYTHON} -m wheel tags --remove \ - --platform-tag "manylinux_${GLIBC_MAJOR}_${GLIBC_MINOR}_x86_64" \ - dist/mkl_umath*.whl - -${PYTHON} -m pip install dist/mkl_umath*.whl \ - --no-build-isolation \ - --no-deps \ - --only-binary :all: \ - --no-index \ - --prefix "${PREFIX}" \ +CC=icx CXX=icpx $PYTHON -m pip install --no-build-isolation --no-deps \ + -Csetup-args="-Dmkl_threading=gnu_thread" \ + -Ccompile-args=-v \ + . \ -vv - -# Copy wheel package -if [ -n "${WHEELS_OUTPUT_FOLDER}" ]; then - mkdir -p "${WHEELS_OUTPUT_FOLDER}" - cp dist/mkl_umath*.whl "${WHEELS_OUTPUT_FOLDER}" -fi diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index 0f279420..c3378fba 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -1,11 +1,22 @@ -numpy: +# Python versions for ci/matrix.py, one per line, without selectors +python: + - "3.10.* *_cpython" + - "3.11.* *_cpython" + - "3.12.* *_cpython" + - "3.13.* *_cp313" + - "3.14.* *_cp314" + - "3.14.* *_cp314t" + - "3.15.* *_cp315" +numpy: # as in conda-forge's global pinning; CI overrides it with --numpy - '2' c_compiler: # [linux] - gcc # [linux] +c_compiler_version: # [linux] + - '15' # [linux] cxx_compiler: # [linux] - gxx # [linux] cxx_compiler_version: # [linux] - - '14' # [linux] + - '15' # [linux] c_stdlib: # [linux] - sysroot # [linux] c_stdlib_version: # [linux] diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 10679606..ef2d494a 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -1,18 +1,22 @@ +{% set version = "0.6.0" %} +{% set buildnumber = 0 %} + package: name: mkl_umath - version: {{ GIT_DESCRIBE_TAG }} + version: {{ version }} source: path: ../ build: - number: {{ GIT_DESCRIBE_NUMBER }} - script_env: - - WHEELS_OUTPUT_FOLDER + number: {{ buildnumber }} + skip: true # [osx] + ignore_run_exports_from: + - {{ compiler('dpcpp') }} # [linux] ignore_run_exports: - blas - - numpy - - intel-openmp # [win] + missing_dso_whitelist: + - $RPATH/mkl_umath_loops.dll # [win] requirements: build: @@ -27,24 +31,23 @@ requirements: - git - cython >=3.1.0 - python - - python-build >=1.2.2 - mkl-devel - - intel-openmp - - intel-cmplr-lib-rt + - intel-cmplr-lib-rt # [not osx] - numpy - - wheel >=0.41.3 + - llvm-openmp - pip run: - python - - {{ pin_compatible('mkl') }} - - {{ pin_compatible('intel-openmp') }} - - {{ pin_compatible('intel-cmplr-lib-rt') }} + # NumPy 1 vs 2 loops are chosen at compile time (USE_NUMPY_2), so a build + # against NumPy 2 must not run on NumPy 1.x; run_exports add the upper bound - numpy >=2 test: requires: + - pip - pytest commands: + - pip check - pytest --pyargs mkl_umath imports: - mkl_umath diff --git a/meson.build b/meson.build index c9312856..10c83133 100644 --- a/meson.build +++ b/meson.build @@ -112,6 +112,13 @@ else error('Unsupported system.') endif +# mkl-devel's CMake config is under \Library on Windows, which CMake only +# finds through a Library\bin PATH entry: a conda env has one, a venv doesn't +mkl_cmake_prefix = [] +if host_machine.system() == 'windows' + mkl_cmake_prefix += '-DCMAKE_PREFIX_PATH=' + (py.get_path('data') / 'Library') +endif + mkl_dep = dependency('MKL', method: 'cmake', modules: ['MKL::MKL'], cmake_args: [ @@ -119,7 +126,7 @@ mkl_dep = dependency('MKL', method: 'cmake', '-DMKL_LINK=dynamic', '-DMKL_THREADING=' + get_option('mkl_threading'), '-DMKL_INTERFACE=lp64' - ], + ] + mkl_cmake_prefix, required: true ) diff --git a/mkl_umath/tests/AGENTS.md b/mkl_umath/tests/AGENTS.md index a9f4d5c9..f78272b8 100644 --- a/mkl_umath/tests/AGENTS.md +++ b/mkl_umath/tests/AGENTS.md @@ -19,7 +19,7 @@ pytest mkl_umath/tests/ ``` ## CI integration -- Tests run in conda-package.yml workflow +- Tests run in the conda-package-cf.yml and intel-ci.yml workflows - Separate test jobs per Python version (3.10-3.15) - Linux + Windows platforms diff --git a/pyproject.toml b/pyproject.toml index ec1197ae..89eeae09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ requires = [ "ninja", "Cython>=3.1.0", "numpy>=2", - "mkl-devel", + "mkl-devel>=2026.2,<2027", "cmake" ] @@ -80,9 +80,33 @@ extend-exclude = "(^|/)_vendored/" force-exclude = "(^|/)_vendored/" line-length = 80 -[tool.cython-lint] -ignore = ['E722'] # do not use bare 'except' -max-line-length = 80 +[tool.cibuildwheel] +build-verbosity = 1 + +[tool.cibuildwheel.linux] +# Intel's icx, as the conda recipe builds with, from Intel's repository +before-all = [ + "rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB", + "printf '[oneAPI]\\nname=Intel oneAPI\\nbaseurl=https://yum.repos.intel.com/oneapi\\nenabled=1\\ngpgcheck=1\\nrepo_gpgcheck=1\\ngpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB\\n' > /etc/yum.repos.d/oneAPI.repo", + "dnf install -y intel-oneapi-compiler-dpcpp-cpp" +] +environment = { CC = "/opt/intel/oneapi/compiler/latest/bin/icx" } +# MKL, OpenMP and icx's runtime come from wheels at run time, see [tool.gh-ci] +# Tagged as the mkl wheel is, as auditwheel doesn't check the excluded libraries +repair-wheel-command = "auditwheel repair --plat manylinux_2_28_x86_64 --only-plat --exclude libmkl_intel_lp64.so.3 --exclude libmkl_intel_thread.so.3 --exclude libmkl_core.so.3 --exclude libiomp5.so --exclude libimf.so --exclude libsvml.so --exclude libirng.so --exclude libintlc.so.5 -w {dest_dir} {wheel}" + +[tool.cibuildwheel.windows] +# MSVC's environment, as python.org builds CPython with, and icx from conda-forge +before-all = "python {project}/ci/install_icx.py C:/icx" +config-settings = { setup-args = "--vsenv" } +environment = { CC = "C:/icx/Library/bin/icx.exe", LIB = "C:/icx/Library/lib;$LIB", LINK = "/LIBPATH:C:/icx/Library/lib" } +# MKL, OpenMP and icx's runtime come from wheels at run time, see [tool.gh-ci] +repair-wheel-command = "delvewheel repair --no-dll mkl_umath_loops.dll;mkl_intel_thread.3.dll;mkl_core.3.dll;libiomp5md.dll;libmmd.dll;svml_dispmd.dll;libircmd.dll;libirngmd.dll -w {dest_dir} {wheel}" + +[tool.gh-ci] +# What the wheels need at run time, which conda packages get from the recipe; +# the same MKL as [build-system], whose library names the repair commands use +wheel-requires = ["mkl >=2026.2,<2027", "intel-openmp >=2026.2,<2027", "intel-cmplr-lib-rt >=2026.2,<2027"] [tool.isort] ensure_newline_before_comments = true From 6d0e5decc1fc3da65ce83e9ec1f42d6f4c887789 Mon Sep 17 00:00:00 2001 From: "Harlow, Jordan" Date: Fri, 6 Nov 2026 11:30:28 -0700 Subject: [PATCH 2/2] Read x86 and osx64 as conda-build does, and check $ORIGIN exactly conda-build reads x86 as any x86 CPU and has no osx64 selector, while rattler-build has x86 for 32-bit only and osx64 for Intel macOS, so ci/matrix.py now gives conda-build its own values for both and refuses a conda-build skip that uses osx64. ci/build.py keeps a search path entry only when it is $ORIGIN or a directory under it, names RPATH and RUNPATH entries apart, and rewrites the wheels in scratch, so a refused wheel never reaches --output. --- ci/README.md | 2 +- ci/build.py | 43 +++++++++++++++++++++++++++---------------- ci/matrix.py | 42 ++++++++++++++++++++++++++---------------- 3 files changed, 54 insertions(+), 33 deletions(-) diff --git a/ci/README.md b/ci/README.md index 67ea2451..6f7f8fe5 100644 --- a/ci/README.md +++ b/ci/README.md @@ -15,7 +15,7 @@ | `test.py` | Runs the installed package's test suite with pytest. | | `benchmark.py` | Runs the asv suite in `../benchmarks/` against the installed package. | -The scripts need only the Python standard library. The recipe skips macOS, as `conda-forge/mkl_umath-feedstock` does. +The scripts need nothing beyond the Python standard library and the tools each one's docstring names. The recipe skips macOS, as `conda-forge/mkl_umath-feedstock` does. **A pull request into `main`.** Actions builds the oldest and the newest regular Python versions on Linux and Windows, as a conda package and, where `pyproject.toml` declares the platform, as a wheel, and tests each package. diff --git a/ci/build.py b/ci/build.py index a2ee8bf5..652f44c6 100644 --- a/ci/build.py +++ b/ci/build.py @@ -63,7 +63,7 @@ # The ELF program header and dynamic section values the RUNPATH check reads. PT_LOAD, PT_DYNAMIC = 1, 2 DT_NULL, DT_STRTAB, DT_RPATH, DT_RUNPATH = 0, 5, 15, 29 -ORIGIN_PREFIXES = ("$ORIGIN", "${ORIGIN}") +ORIGIN = ("$ORIGIN", "${ORIGIN}") # The rattler-build release fetched when none is on PATH, per system, with the # file name it goes under and the sha256 its release publishes. @@ -152,8 +152,8 @@ def record_hash(data: bytes) -> str: def elf_search_paths(data: bytes, name: str) -> list[tuple[int, int, str]]: """ - The RPATH and RUNPATH strings of an ELF file, as (file offset, length, - value); none for a file that is not ELF. + The RPATH and RUNPATH strings of an ELF file, as (RPATH or RUNPATH, file + offset, length, value); none for a file that is not ELF. """ if data[:4] != b"\x7fELF": return [] @@ -187,28 +187,38 @@ def elf_search_paths(data: bytes, name: str) -> list[tuple[int, int, str]]: if tag in (DT_RPATH, DT_RUNPATH): start = base + value end = data.index(b"\0", start) - found.append((start, end - start, data[start:end].decode())) + kind = "RPATH" if tag == DT_RPATH else "RUNPATH" + found.append((kind, start, end - start, data[start:end].decode())) return found +def origin_relative(path: str) -> bool: + """Whether a search path entry is $ORIGIN or a directory under it.""" + return any( + path == token or path.startswith(token + "/") for token in ORIGIN + ) + + def origin_only(data: bytes, name: str) -> tuple[bytes, list[str]]: """ An ELF file with its RPATH and RUNPATH cut to their $ORIGIN entries, in place as patchelf does, and the entries it dropped. """ edited, dropped = bytearray(data), [] - for start, length, value in elf_search_paths(data, name): + for kind, start, length, value in elf_search_paths(data, name): paths = value.split(":") - kept = [path for path in paths if path.startswith(ORIGIN_PREFIXES)] + kept = [path for path in paths if origin_relative(path)] if kept == paths: continue - # An empty RUNPATH would mean the working directory + # An empty search path would mean the working directory if not kept: - raise BuildError(f"{name}: no $ORIGIN entry in RUNPATH {value!r}") + raise BuildError(f"{name}: no $ORIGIN entry in {kind} {value!r}") edited[start : start + length] = ( ":".join(kept).encode().ljust(length, b"\0") ) - dropped += [path for path in paths if path not in kept] + dropped += [ + f"{kind} entry {path}" for path in paths if path not in kept + ] return bytes(edited), dropped @@ -393,6 +403,7 @@ def main(argv: list[str] | None = None) -> int: ) if system == "rattler-build": ensure_rattler_build(args.tools) + requires = wheel_requires() if system == "wheel" else [] output = args.output.resolve() with tempfile.TemporaryDirectory() as scratch: for command in commands( @@ -404,14 +415,14 @@ def main(argv: list[str] | None = None) -> int: ): print(f"+ {shlex.join(command)}", flush=True) subprocess.run(command, check=True) + # Rewritten in scratch, so a refused wheel never reaches the output + if system == "wheel": + for wheel in sorted(Path(scratch).rglob("*.whl")): + for dropped in rewrite_wheel(wheel, requires): + print(f"build: dropped {dropped}") built = collect(Path(scratch), output) - if system == "wheel": - requires = wheel_requires() - for wheel in built: - for dropped in rewrite_wheel(wheel, requires): - print(f"build: dropped RUNPATH entry {dropped}") - if requires: - print("build: added Requires-Dist " + ", ".join(requires)) + if requires: + print("build: added Requires-Dist " + ", ".join(requires)) print("build: " + ", ".join(path.name for path in built)) return 0 diff --git a/ci/matrix.py b/ci/matrix.py index 5b197bd7..b4fc8597 100644 --- a/ci/matrix.py +++ b/ci/matrix.py @@ -77,8 +77,8 @@ # The platforms the workflow can build on: GitHub-hosted runners have all # three. A declared platform missing here is reported and left out. BUILT_PLATFORMS = ("linux", "windows", "macos") -# What a platform-only selector means on each platform, for both conda-build and -# rattler-build. +# What a platform-only selector means on each platform, as rattler-build defines +# it; CONDA_BUILD_SELECTORS holds where conda-build reads it differently. SELECTORS = { "linux": { "linux": True, @@ -130,6 +130,9 @@ "s390x": False, }, } +# conda-build reads x86 as any x86 CPU and has no osx64; None refuses a skip +# that uses it. +CONDA_BUILD_SELECTORS = {"x86": True, "osx64": None} CONDA_SKIP_RE = re.compile(r"\s*skip\s*:") # conda-build's own selector pattern, which lets a comment follow the selector CONDA_SELECTOR_RE = re.compile(r"(.+?)\s*(#.*)?\[([^\[\]]+)\](?(2)[^\(\)]*)$") @@ -333,12 +336,15 @@ def from_pyproject(path: Path) -> list[Python]: return [Python(version, version) for version in versions] -def skips_platform(expression: str, platform: str) -> bool | None: +def skips_platform(expression: str, platform: str, system: str) -> bool | None: """ - Whether a skip expression rules out a whole platform: True or False when it - depends on the platform alone, None when it also depends on anything else, - such as the Python version. + Whether a skip expression rules out a whole platform, as the build system + reads it: True or False when it depends on the platform alone, None when it + also depends on anything else, such as the Python version. """ + names = SELECTORS[platform] | ( + CONDA_BUILD_SELECTORS if system == "conda-build" else {} + ) try: tree = ast.parse( expression.strip().removeprefix("${{").removesuffix("}}").strip(), @@ -353,8 +359,8 @@ def value(node: ast.AST) -> bool: return all(values) if isinstance(node.op, ast.And) else any(values) if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): return not value(node.operand) - if isinstance(node, ast.Name) and node.id in SELECTORS[platform]: - return SELECTORS[platform][node.id] + if isinstance(node, ast.Name) and names.get(node.id) is not None: + return names[node.id] if isinstance(node, ast.Constant) and isinstance(node.value, bool): return node.value raise ValueError(ast.dump(node)) @@ -365,23 +371,25 @@ def value(node: ast.AST) -> bool: return None -def platforms_without(skips: list[str], where: str) -> list[str]: +def platforms_without(skips: list[str], where: str, system: str) -> list[str]: """ Every platform in PLATFORMS, but those a skip rules out; a skip that - depends on more than the platform is refused. + depends on more than the platform, or on a name the build system doesn't + define, is refused. """ for skip in skips: if any( - skips_platform(skip, platform) is None for platform in PLATFORMS + skips_platform(skip, platform, system) is None + for platform in PLATFORMS ): raise DeclarationError( - f"{where}: the skip {skip!r} depends on more than the " - "platform, so the matrix would still list the builds it skips" + f"{where}: cannot tell which platforms the skip {skip!r} rules " + "out, so the matrix would still list the builds it skips" ) kept = [ platform for platform in PLATFORMS - if not any(skips_platform(skip, platform) for skip in skips) + if not any(skips_platform(skip, platform, system) for skip in skips) ] if not kept: raise DeclarationError( @@ -454,7 +462,9 @@ def from_rattler_recipe(recipe: Path) -> list[str]: else str(recipe) ) return platforms_without( - yaml_build_skips(recipe.read_text(encoding="utf-8"), where), where + yaml_build_skips(recipe.read_text(encoding="utf-8"), where), + where, + "rattler-build", ) @@ -487,7 +497,7 @@ def from_conda_recipe(recipe: Path) -> list[str]: "skip: true # [selector]" ) skips.append(selector) - return platforms_without(skips, where) + return platforms_without(skips, where, "conda-build") def from_os_classifiers(path: Path) -> list[str]: