From 372d34d9dcd23f42cc85ba1f82f4baae01845fdd Mon Sep 17 00:00:00 2001 From: Larry Gritz Date: Fri, 16 Oct 2026 00:34:00 -0700 Subject: [PATCH] ci: Cache the dependencies that Windows CI builds from source This brings the Windows CI jobs down from around 30m to around 10m (for the 2nd and subsequent build on the same branch or PR). TL;DR: On Linux and Mac, our CI uses GHA cache action, in conjunction with ccache, to save and restore the ccache area, thus making recompiles (including run to run on the same branch and with the same flags) much less expensive, both for OSL itself and also for any dependencies that we rebuild from scratch. CMake's Visual Studio generator ignores CMAKE_CXX_COMPILER_LAUNCHER, so ccache isn't used for our Windows CI. We could switch to Ninja and use ccache, but then there is a testing gap where we aren't building in the way that most Windows users will. So we don't cache, and thus have long build times. This PR caches the installed results of our dependency builds on Windows, saving about 2/3 of the total time. The dependencies we build in Windows CI (and now cache for most runs) are zlib, libpng, pybind11, OpenEXR, pugixml, OpenColorIO, and OpenImageIO. Assisted-by: Claude Code / Claude Opus 5.5 Signed-off-by: Larry Gritz --- .github/workflows/build-steps.yml | 28 ++++++++++ src/build-scripts/gh-win-deps-cache-key.bash | 57 ++++++++++++++++++++ src/build-scripts/gh-win-installdeps.bash | 32 ++++++++--- 3 files changed, 109 insertions(+), 8 deletions(-) create mode 100755 src/build-scripts/gh-win-deps-cache-key.bash diff --git a/.github/workflows/build-steps.yml b/.github/workflows/build-steps.yml index 1ea0184b8..ecec1f4d3 100644 --- a/.github/workflows/build-steps.yml +++ b/.github/workflows/build-steps.yml @@ -181,6 +181,24 @@ jobs: with: path: ~/llvm/llvm-build key: llvm-windows-${{ env.LLVM_GOOGLE_DRIVE_ID }} + - name: deps-cache-key + # Windows CI builds several dependencies from source, which takes + # most of the job's time. Cache the installed results. + id: deps-cache-key + if: runner.os == 'Windows' + shell: bash + run: | + key=$(src/build-scripts/gh-win-deps-cache-key.bash deps-${{inputs.nametag}}) + echo "key=$key" >> $GITHUB_OUTPUT + - name: deps-cache-restore + id: deps-cache-restore + if: runner.os == 'Windows' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/ext/dist + ext/dist + key: ${{ steps.deps-cache-key.outputs.key }} - name: Install gcc if: inputs.gcc_action_ver != '' run: | @@ -194,6 +212,8 @@ jobs: python-version: ${{ inputs.python_action_ver }} - name: Dependencies shell: bash + env: + OSL_DEPS_CACHE_HIT: ${{ steps.deps-cache-restore.outputs.cache-hit }} run: | eval "$DEPCMDS" if [[ "$RUNNER_OS" == "Linux" ]]; then @@ -217,6 +237,14 @@ jobs: with: path: ~/llvm/llvm-build key: llvm-windows-${{ env.LLVM_GOOGLE_DRIVE_ID }} + - name: deps-cache-save + if: runner.os == 'Windows' && steps.deps-cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/ext/dist + ext/dist + key: ${{ steps.deps-cache-key.outputs.key }} - name: Install sonar-scanner and build-wrapper if: inputs.sonar == '1' uses: sonarsource/sonarqube-scan-action@22918119ff8e1ca75a623e15c8296b6ea4fbe28f # v8.2.1 diff --git a/src/build-scripts/gh-win-deps-cache-key.bash b/src/build-scripts/gh-win-deps-cache-key.bash new file mode 100755 index 000000000..22d9c6ef1 --- /dev/null +++ b/src/build-scripts/gh-win-deps-cache-key.bash @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Copyright Contributors to the Open Shading Language project. +# SPDX-License-Identifier: BSD-3-Clause +# https://github.com/AcademySoftwareFoundation/OpenShadingLanguage + +# Print a cache key for the dependencies that gh-win-installdeps.bash +# builds from source. The key changes when anything that affects those +# builds changes: the dependency versions, the build settings, or the +# build scripts. Branch names (like OIIO's "release") are resolved to their +# current commit, so the deps are rebuilt when the branch moves. +# +# Usage: gh-win-deps-cache-key.bash + +set -e + +# Resolve a branch name to its current commit. Tags and commit hashes are +# printed unchanged. +resolve_ref () { + local repo=$1 ref=$2 + if [[ "$ref" == "" ]] ; then + return + fi + # If ls-remote fails, fail the script rather than keying on a branch + # name, which could keep restoring stale builds. + local refs + refs=$(git ls-remote "$repo" "refs/heads/$ref") || exit 1 + local sha=$(echo "$refs" | cut -f1) + echo "${sha:-$ref}" +} + +openexr=$(resolve_ref ${OPENEXR_REPO:-https://github.com/AcademySoftwareFoundation/openexr.git} ${OPENEXR_VERSION}) +opencolorio=$(resolve_ref ${OPENCOLORIO_REPO:-https://github.com/AcademySoftwareFoundation/OpenColorIO.git} ${OPENCOLORIO_VERSION}) +openimageio=$(resolve_ref https://github.com/${OPENIMAGEIO_REPO:-AcademySoftwareFoundation/OpenImageIO} ${OPENIMAGEIO_VERSION}) +scripts=$(cat src/build-scripts/gh-win-installdeps.bash src/build-scripts/build_*.bash | sha256sum) + +desc=$( + echo "zlib=${ZLIB_VERSION}" + echo "libpng=${LIBPNG_VERSION}" + echo "pybind11=${PYBIND11_VERSION}" + echo "pugixml=${PUGIXML_VERSION}" + echo "openexr=${openexr}" + echo "opencolorio=${opencolorio}" + echo "openimageio=${openimageio}" + echo "OpenImageIO_BUILD_MISSING_DEPS=${OpenImageIO_BUILD_MISSING_DEPS}" + echo "OPENIMAGEIO_UNITY=${OPENIMAGEIO_UNITY}" + echo "CMAKE_GENERATOR=${CMAKE_GENERATOR}" + echo "CMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}" + echo "PYTHON_VERSION=${PYTHON_VERSION}" + echo "USE_SIMD=${USE_SIMD}" + echo "scripts=${scripts}" +) + +# Show the inputs in the log, to help explain cache misses +echo "$desc" | sed 's/^/ /' >&2 + +echo "$1-$(echo "$desc" | sha256sum | cut -c1-16)" diff --git a/src/build-scripts/gh-win-installdeps.bash b/src/build-scripts/gh-win-installdeps.bash index a842967bc..3c214e807 100755 --- a/src/build-scripts/gh-win-installdeps.bash +++ b/src/build-scripts/gh-win-installdeps.bash @@ -71,15 +71,30 @@ vcpkg list ######################################################################## # Dependency method #2: Build from source ourselves # +# If the CI workflow restored the built dependencies from its cache (see +# gh-win-deps-cache-key.bash), OSL_DEPS_CACHE_HIT is "true" and we skip +# the builds, only setting the variables that the build scripts export. # -src/build-scripts/build_zlib.bash -export ZLIB_ROOT=$PWD/ext/dist +# build_dep SCRIPT ROOTVAR : source a dependency build script, or on a +# cache hit, just set ROOTVAR to where the build would have installed it. +build_dep () { + if [[ "$OSL_DEPS_CACHE_HIT" == "true" ]] ; then + echo "Using cached build from $1" + export $2=$LOCAL_DEPS_DIR/dist + else + source src/build-scripts/$1 + fi +} -src/build-scripts/build_libpng.bash +if [[ "$OSL_DEPS_CACHE_HIT" != "true" ]] ; then + src/build-scripts/build_zlib.bash + src/build-scripts/build_libpng.bash +fi +export ZLIB_ROOT=$PWD/ext/dist export PNG_ROOT=$PWD/ext/dist -source src/build-scripts/build_pybind11.bash +build_dep build_pybind11.bash pybind11_ROOT export pybind11_ROOT=$PWD/ext/dist # nanobind is only needed for the non-default python binding backends. Install @@ -90,16 +105,16 @@ fi if [[ "$OPENEXR_VERSION" != "" ]] ; then - source src/build-scripts/build_openexr.bash + build_dep build_openexr.bash OpenEXR_ROOT fi if [[ "$PUGIXML_VERSION" != "" ]] ; then - source src/build-scripts/build_pugixml.bash + build_dep build_pugixml.bash pugixml_ROOT export OSL_CMAKE_FLAGS+=" -DUSE_EXTERNAL_PUGIXML=1 " fi if [[ "$OPENCOLORIO_VERSION" != "" ]] ; then - source src/build-scripts/build_opencolorio.bash + build_dep build_opencolorio.bash OpenColorIO_ROOT fi cp $DEP_DIR/lib/*.lib $DEP_DIR/bin || true @@ -126,7 +141,8 @@ if [[ "$OPENIMAGEIO_VERSION" != "" ]] ; then # only a savings in CI where there are only 1-2 cores available.) export OPENIMAGEIO_CMAKE_FLAGS+=" -DCMAKE_UNITY_BUILD=ON -DCMAKE_UNITY_BUILD_MODE=BATCH" fi - source src/build-scripts/build_openimageio.bash + build_dep build_openimageio.bash OpenImageIO_ROOT + export PYTHONPATH=$OpenImageIO_ROOT/lib/python${PYTHON_VERSION}/site-packages:$PYTHONPATH fi cp $DEP_DIR/lib/*.lib $DEP_DIR/bin || true