From 99f7d8676d05f69f537ba1b10d28bd181f74851e Mon Sep 17 00:00:00 2001 From: Larry Gritz Date: Fri, 16 Oct 2026 00:30:35 -0700 Subject: [PATCH] ci: Cache Windows LLVM download, and retry curl downloads Our Windows CI often fails suriously in ways that are unrelated to whatever changes are in a PR, and then succeeds upon retry (sometimes after multiple retries). Often it's at the step where it's downloading the pre-built Windows LLVM that we stashed on a Google drive (we could not find a pre-built one with exactly the options we wanted, and we can't build LLM from scratch in a reasonable time on the CI machines). I finally (with some help from Claude) tracked down the source of these failures: we're hitting Google Drive download limits! This patch uses GHA cache action to reuse those downloads, generally only retrieving from the Google drive the first time a branch or PR goes through CI, which throttles usage enough to keep us under quota. The specifics: * Cache the unpacked LLVM (~/llvm/llvm-build) with actions/cache, keyed on LLVM_GOOGLE_DRIVE_ID. Both Windows jobs share the entry. The save step runs right after Dependencies, so a later build failure doesn't lose it. gh-win-installdeps.bash skips the download when the cache restored the directory. * When we do download, check the result with `unzip -t`. If it is not a zip, fail with a clear message and print the start of the file. * Add `--fail --retry 5 --retry-delay 10` to the curl downloads in gh-win-installdeps.bash, build_llvm.bash, and build_cmake.bash, to ride out transient network failures. * Fix a missing `;` before `then` in gh-win-installdeps.bash. Assisted-by: Claude Code / Claude Opus 5.5 Signed-off-by: Larry Gritz --- .github/workflows/build-steps.yml | 15 ++++++++++ src/build-scripts/build_cmake.bash | 4 +-- src/build-scripts/build_llvm.bash | 4 +-- src/build-scripts/gh-win-installdeps.bash | 35 ++++++++++++++++++----- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-steps.yml b/.github/workflows/build-steps.yml index d41250d471..1ea0184b81 100644 --- a/.github/workflows/build-steps.yml +++ b/.github/workflows/build-steps.yml @@ -172,6 +172,15 @@ jobs: path: ${{ env.CCACHE_DIR }} key: ${{inputs.nametag}}-${{steps.ccache_cache_keys.outputs.date}} restore-keys: ${{inputs.nametag}} + - name: llvm-cache-restore + # Windows CI downloads a prebuilt LLVM from Google Drive, which has a + # per-file download quota. Cache it so most runs skip the download. + id: llvm-cache-restore + if: runner.os == 'Windows' && env.LLVM_GOOGLE_DRIVE_ID != '' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/llvm/llvm-build + key: llvm-windows-${{ env.LLVM_GOOGLE_DRIVE_ID }} - name: Install gcc if: inputs.gcc_action_ver != '' run: | @@ -202,6 +211,12 @@ jobs: elif [[ "$RUNNER_OS" == "Windows" ]]; then src/build-scripts/gh-win-installdeps.bash fi + - name: llvm-cache-save + if: runner.os == 'Windows' && env.LLVM_GOOGLE_DRIVE_ID != '' && steps.llvm-cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/llvm/llvm-build + key: llvm-windows-${{ env.LLVM_GOOGLE_DRIVE_ID }} - name: Install sonar-scanner and build-wrapper if: inputs.sonar == '1' uses: sonarsource/sonarqube-scan-action@22918119ff8e1ca75a623e15c8296b6ea4fbe28f # v8.2.1 diff --git a/src/build-scripts/build_cmake.bash b/src/build-scripts/build_cmake.bash index 942d755790..49d4c56733 100755 --- a/src/build-scripts/build_cmake.bash +++ b/src/build-scripts/build_cmake.bash @@ -18,14 +18,14 @@ CMAKE_INSTALL_DIR=${CMAKE_INSTALL_DIR:=${LOCAL_DEPS_DIR}/cmake} if [[ `uname` == "Linux" && `uname -m` == "x86_64" ]] ; then mkdir -p ${CMAKE_INSTALL_DIR} || true - curl --location "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh" -o "cmake.sh" + curl --fail --retry 5 --retry-delay 10 --location "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh" -o "cmake.sh" sh cmake.sh --skip-license --prefix=${CMAKE_INSTALL_DIR} export PATH=${CMAKE_INSTALL_DIR}/bin:$PATH fi if [[ `uname` == "Linux" && `uname -m` == "aarch64" ]] ; then mkdir -p ${CMAKE_INSTALL_DIR} || true - curl --location "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-aarch64.sh" -o "cmake.sh" + curl --fail --retry 5 --retry-delay 10 --location "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-aarch64.sh" -o "cmake.sh" sh cmake.sh --skip-license --prefix=${CMAKE_INSTALL_DIR} export PATH=${CMAKE_INSTALL_DIR}/bin:$PATH diff --git a/src/build-scripts/build_llvm.bash b/src/build-scripts/build_llvm.bash index e2d443b2a7..ae1ff5738f 100755 --- a/src/build-scripts/build_llvm.bash +++ b/src/build-scripts/build_llvm.bash @@ -20,7 +20,7 @@ if [[ `uname` == "Linux" ]] ; then : ${LLVM_DISTRO_NAME:=ubuntu-18.04} LLVMTAR=clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-${LLVM_DISTRO_NAME}.tar.xz echo LLVMTAR = $LLVMTAR - curl --location https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/${LLVMTAR} -o $LLVMTAR + curl --fail --retry 5 --retry-delay 10 --location https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/${LLVMTAR} -o $LLVMTAR ls -l $LLVMTAR tar xf $LLVMTAR rm -f $LLVMTAR @@ -30,7 +30,7 @@ elif [[ `uname -s` == "Windows" || "${RUNNER_OS}" == "Windows" ]] ; then : ${LLVM_DISTRO_NAME:=ubuntu-18.04} LLVMTAR=clang+llvm-${LLVM_VERSION}-x86_64-pc-windows-msvc.tar.xz echo LLVMTAR = $LLVMTAR - curl --location https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/${LLVMTAR} -o $LLVMTAR + curl --fail --retry 5 --retry-delay 10 --location https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/${LLVMTAR} -o $LLVMTAR ls -l $LLVMTAR tar xf $LLVMTAR rm -f $LLVMTAR diff --git a/src/build-scripts/gh-win-installdeps.bash b/src/build-scripts/gh-win-installdeps.bash index 520c72df24..a842967bc6 100755 --- a/src/build-scripts/gh-win-installdeps.bash +++ b/src/build-scripts/gh-win-installdeps.bash @@ -13,6 +13,10 @@ mkdir -p "$DEP_DIR" mkdir -p ext VCPKG_INSTALLATION_ROOT=/c/vcpkg +# Retry downloads a few times to ride out transient network failures, and +# fail on HTTP errors rather than saving an error page as the download. +CURL_RETRY_FLAGS="--fail --retry 5 --retry-delay 10" + export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:=.} export CMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH;$DEP_DIR" export CMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH;$VCPKG_INSTALLATION_ROOT/installed/x64-windows-release" @@ -133,14 +137,31 @@ ls -R -l "$DEP_DIR" if [[ "$LLVM_VERSION" != "" ]] ; then echo "Running llvm_build.bash" source src/build-scripts/build_llvm.bash -elif [[ "$LLVM_GOOGLE_DRIVE_ID" != "" ]] then - echo "Installing LLVM from google drive" +elif [[ "$LLVM_GOOGLE_DRIVE_ID" != "" ]] ; then mkdir -p $HOME/llvm pushd $HOME/llvm - #LLVM_GOOGLE_DRIVE_ID="1uy7PNVlTQ-H56unXGOS6siRWtNcdS1J7" - LLVM_ZIP_FILENAME=llvm-build.zip - time curl --insecure -L "https://drive.usercontent.google.com/download?id=${LLVM_GOOGLE_DRIVE_ID}&confirm=xxx" -o $LLVM_ZIP_FILENAME - unzip $LLVM_ZIP_FILENAME > /dev/null + # The llvm-build dir may already be present if restored by actions/cache + # (keyed on LLVM_GOOGLE_DRIVE_ID), in which case skip the download. This + # also keeps us from tripping Google Drive's per-file download quota. + if [[ -d llvm-build ]] ; then + echo "Using cached LLVM from google drive id ${LLVM_GOOGLE_DRIVE_ID}" + else + echo "Installing LLVM from google drive" + #LLVM_GOOGLE_DRIVE_ID="1uy7PNVlTQ-H56unXGOS6siRWtNcdS1J7" + LLVM_ZIP_FILENAME=llvm-build.zip + time curl $CURL_RETRY_FLAGS --insecure -L "https://drive.usercontent.google.com/download?id=${LLVM_GOOGLE_DRIVE_ID}&confirm=xxx" -o $LLVM_ZIP_FILENAME + # When the download quota is exceeded, Google Drive returns an HTML + # error page with HTTP status 200, so check that we got a real zip. + if ! unzip -tq $LLVM_ZIP_FILENAME > /dev/null ; then + echo "ERROR: LLVM download from google drive is not a valid zip file." + echo "Google Drive may have exceeded its download quota. Contents:" + head -c 2000 $LLVM_ZIP_FILENAME + echo + exit 1 + fi + unzip $LLVM_ZIP_FILENAME > /dev/null + rm -f $LLVM_ZIP_FILENAME + fi export LLVM_ROOT=$PWD/llvm-build popd else @@ -152,7 +173,7 @@ echo "LLVM_ROOT = $LLVM_ROOT" mkdir -p winflexbison pushd winflexbison WFBZIP=win_flex_bison-2.5.25.zip -curl --location https://github.com/lexxmark/winflexbison/releases/download/v2.5.25/$WFBZIP -o $WFBZIP +curl $CURL_RETRY_FLAGS --location https://github.com/lexxmark/winflexbison/releases/download/v2.5.25/$WFBZIP -o $WFBZIP unzip $WFBZIP export FLEX_ROOT=$PWD export BISON_ROOT=$PWD