diff --git a/.github/workflows/build-steps.yml b/.github/workflows/build-steps.yml index d41250d47..1ea0184b8 100644 --- a/.github/workflows/build-steps.yml +++ b/.github/workflows/build-steps.yml @@ -172,6 +172,15 @@ jobs: path: ${{ env.CCACHE_DIR }} key: ${{inputs.nametag}}-${{steps.ccache_cache_keys.outputs.date}} restore-keys: ${{inputs.nametag}} + - name: llvm-cache-restore + # Windows CI downloads a prebuilt LLVM from Google Drive, which has a + # per-file download quota. Cache it so most runs skip the download. + id: llvm-cache-restore + if: runner.os == 'Windows' && env.LLVM_GOOGLE_DRIVE_ID != '' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/llvm/llvm-build + key: llvm-windows-${{ env.LLVM_GOOGLE_DRIVE_ID }} - name: Install gcc if: inputs.gcc_action_ver != '' run: | @@ -202,6 +211,12 @@ jobs: elif [[ "$RUNNER_OS" == "Windows" ]]; then src/build-scripts/gh-win-installdeps.bash fi + - name: llvm-cache-save + if: runner.os == 'Windows' && env.LLVM_GOOGLE_DRIVE_ID != '' && steps.llvm-cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/llvm/llvm-build + key: llvm-windows-${{ env.LLVM_GOOGLE_DRIVE_ID }} - name: Install sonar-scanner and build-wrapper if: inputs.sonar == '1' uses: sonarsource/sonarqube-scan-action@22918119ff8e1ca75a623e15c8296b6ea4fbe28f # v8.2.1 diff --git a/src/build-scripts/build_cmake.bash b/src/build-scripts/build_cmake.bash index 942d75579..49d4c5673 100755 --- a/src/build-scripts/build_cmake.bash +++ b/src/build-scripts/build_cmake.bash @@ -18,14 +18,14 @@ CMAKE_INSTALL_DIR=${CMAKE_INSTALL_DIR:=${LOCAL_DEPS_DIR}/cmake} if [[ `uname` == "Linux" && `uname -m` == "x86_64" ]] ; then mkdir -p ${CMAKE_INSTALL_DIR} || true - curl --location "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh" -o "cmake.sh" + curl --fail --retry 5 --retry-delay 10 --location "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh" -o "cmake.sh" sh cmake.sh --skip-license --prefix=${CMAKE_INSTALL_DIR} export PATH=${CMAKE_INSTALL_DIR}/bin:$PATH fi if [[ `uname` == "Linux" && `uname -m` == "aarch64" ]] ; then mkdir -p ${CMAKE_INSTALL_DIR} || true - curl --location "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-aarch64.sh" -o "cmake.sh" + curl --fail --retry 5 --retry-delay 10 --location "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-aarch64.sh" -o "cmake.sh" sh cmake.sh --skip-license --prefix=${CMAKE_INSTALL_DIR} export PATH=${CMAKE_INSTALL_DIR}/bin:$PATH diff --git a/src/build-scripts/build_llvm.bash b/src/build-scripts/build_llvm.bash index e2d443b2a..ae1ff5738 100755 --- a/src/build-scripts/build_llvm.bash +++ b/src/build-scripts/build_llvm.bash @@ -20,7 +20,7 @@ if [[ `uname` == "Linux" ]] ; then : ${LLVM_DISTRO_NAME:=ubuntu-18.04} LLVMTAR=clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-${LLVM_DISTRO_NAME}.tar.xz echo LLVMTAR = $LLVMTAR - curl --location https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/${LLVMTAR} -o $LLVMTAR + curl --fail --retry 5 --retry-delay 10 --location https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/${LLVMTAR} -o $LLVMTAR ls -l $LLVMTAR tar xf $LLVMTAR rm -f $LLVMTAR @@ -30,7 +30,7 @@ elif [[ `uname -s` == "Windows" || "${RUNNER_OS}" == "Windows" ]] ; then : ${LLVM_DISTRO_NAME:=ubuntu-18.04} LLVMTAR=clang+llvm-${LLVM_VERSION}-x86_64-pc-windows-msvc.tar.xz echo LLVMTAR = $LLVMTAR - curl --location https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/${LLVMTAR} -o $LLVMTAR + curl --fail --retry 5 --retry-delay 10 --location https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/${LLVMTAR} -o $LLVMTAR ls -l $LLVMTAR tar xf $LLVMTAR rm -f $LLVMTAR diff --git a/src/build-scripts/gh-win-installdeps.bash b/src/build-scripts/gh-win-installdeps.bash index 520c72df2..a842967bc 100755 --- a/src/build-scripts/gh-win-installdeps.bash +++ b/src/build-scripts/gh-win-installdeps.bash @@ -13,6 +13,10 @@ mkdir -p "$DEP_DIR" mkdir -p ext VCPKG_INSTALLATION_ROOT=/c/vcpkg +# Retry downloads a few times to ride out transient network failures, and +# fail on HTTP errors rather than saving an error page as the download. +CURL_RETRY_FLAGS="--fail --retry 5 --retry-delay 10" + export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:=.} export CMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH;$DEP_DIR" export CMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH;$VCPKG_INSTALLATION_ROOT/installed/x64-windows-release" @@ -133,14 +137,31 @@ ls -R -l "$DEP_DIR" if [[ "$LLVM_VERSION" != "" ]] ; then echo "Running llvm_build.bash" source src/build-scripts/build_llvm.bash -elif [[ "$LLVM_GOOGLE_DRIVE_ID" != "" ]] then - echo "Installing LLVM from google drive" +elif [[ "$LLVM_GOOGLE_DRIVE_ID" != "" ]] ; then mkdir -p $HOME/llvm pushd $HOME/llvm - #LLVM_GOOGLE_DRIVE_ID="1uy7PNVlTQ-H56unXGOS6siRWtNcdS1J7" - LLVM_ZIP_FILENAME=llvm-build.zip - time curl --insecure -L "https://drive.usercontent.google.com/download?id=${LLVM_GOOGLE_DRIVE_ID}&confirm=xxx" -o $LLVM_ZIP_FILENAME - unzip $LLVM_ZIP_FILENAME > /dev/null + # The llvm-build dir may already be present if restored by actions/cache + # (keyed on LLVM_GOOGLE_DRIVE_ID), in which case skip the download. This + # also keeps us from tripping Google Drive's per-file download quota. + if [[ -d llvm-build ]] ; then + echo "Using cached LLVM from google drive id ${LLVM_GOOGLE_DRIVE_ID}" + else + echo "Installing LLVM from google drive" + #LLVM_GOOGLE_DRIVE_ID="1uy7PNVlTQ-H56unXGOS6siRWtNcdS1J7" + LLVM_ZIP_FILENAME=llvm-build.zip + time curl $CURL_RETRY_FLAGS --insecure -L "https://drive.usercontent.google.com/download?id=${LLVM_GOOGLE_DRIVE_ID}&confirm=xxx" -o $LLVM_ZIP_FILENAME + # When the download quota is exceeded, Google Drive returns an HTML + # error page with HTTP status 200, so check that we got a real zip. + if ! unzip -tq $LLVM_ZIP_FILENAME > /dev/null ; then + echo "ERROR: LLVM download from google drive is not a valid zip file." + echo "Google Drive may have exceeded its download quota. Contents:" + head -c 2000 $LLVM_ZIP_FILENAME + echo + exit 1 + fi + unzip $LLVM_ZIP_FILENAME > /dev/null + rm -f $LLVM_ZIP_FILENAME + fi export LLVM_ROOT=$PWD/llvm-build popd else @@ -152,7 +173,7 @@ echo "LLVM_ROOT = $LLVM_ROOT" mkdir -p winflexbison pushd winflexbison WFBZIP=win_flex_bison-2.5.25.zip -curl --location https://github.com/lexxmark/winflexbison/releases/download/v2.5.25/$WFBZIP -o $WFBZIP +curl $CURL_RETRY_FLAGS --location https://github.com/lexxmark/winflexbison/releases/download/v2.5.25/$WFBZIP -o $WFBZIP unzip $WFBZIP export FLEX_ROOT=$PWD export BISON_ROOT=$PWD