diff --git a/.github/workflows/test-on-push-and-pr.yml b/.github/workflows/test-on-push-and-pr.yml index 735d33a..d09365d 100644 --- a/.github/workflows/test-on-push-and-pr.yml +++ b/.github/workflows/test-on-push-and-pr.yml @@ -189,6 +189,15 @@ jobs: name: "${{ matrix.distro }} ${{ matrix.distro_version }} / python ${{ matrix.runtime_version }}" steps: + - name: Jitter start (stagger matrix shards to spare public.ecr.aws rate limit) + # Every shard is scheduled within milliseconds of the others and pulls + # its base images from public.ecr.aws inside the same rate-limit window, + # so together they trip the throttle (429 Too Many Requests). + run: | + jitter=$((RANDOM % 61)) + echo "Sleeping ${jitter}s to stagger this shard" + sleep "${jitter}" + - uses: actions/checkout@v4 - name: Extract RIE @@ -215,7 +224,7 @@ jobs: echo "RUN apk add curl" >> "$TMPFILE" fi echo "COPY .scratch/${RIE} /usr/bin/${RIE}" >> "$TMPFILE" - docker build . \ + tests/integration/docker-retry.sh docker build . \ -f "$TMPFILE" \ -t ric-test \ --build-arg RUNTIME_VERSION=${{ matrix.runtime_version }} \ diff --git a/tests/integration/docker-retry.sh b/tests/integration/docker-retry.sh new file mode 100755 index 0000000..9d99143 --- /dev/null +++ b/tests/integration/docker-retry.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Run a docker command, retrying when it fails (e.g. public.ecr.aws throttling +# the base image pulls with "429 Too Many Requests"). +# Usage: docker-retry.sh [args...] + +set -uo pipefail + +# Ladder of base wait times (in seconds) between successive attempts. +# Default: 1 min, 2 min, 5 min — 4 attempts total (initial + 3 retries). +# public.ecr.aws rate-limit windows recover on a minute-scale timer, so short +# exponential backoff tends to exhaust its budget before the slot reopens. +IFS=' ' read -r -a DELAYS <<< "${RETRY_DELAYS:-60 120 300}" +JITTER_MAX="${RETRY_JITTER_MAX:-30}" +MAX_ATTEMPTS=$(( ${#DELAYS[@]} + 1 )) + +if (( $# == 0 )); then + >&2 echo "usage: docker-retry.sh [args...]" + exit 2 +fi + +attempt=1 +while true; do + "$@" + status=$? + if (( status == 0 || attempt >= MAX_ATTEMPTS )); then + exit "$status" + fi + + # Base delay from the ladder + additive jitter in [0, JITTER_MAX] so + # concurrent shards don't retry in the exact same second. + base=${DELAYS[$((attempt - 1))]} + jitter=$(( RANDOM % (JITTER_MAX + 1) )) + delay=$(( base + jitter )) + + >&2 echo "docker-retry: '$*' failed (exit ${status}); attempt ${attempt}/${MAX_ATTEMPTS}, retrying in ${delay}s (${base}s + ${jitter}s jitter)." + sleep "$delay" + attempt=$(( attempt + 1 )) +done