diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index a675a7f..dc5443c 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,5 +1,6 @@ self-hosted-runner: labels: + - cs-large-7c14g - amd64-mo-shanghai-8c16g - amd64-mo-guangzhou-2xlarge16 - amd64-mo-guangzhou-2xlarge32 diff --git a/.github/workflows/check-action-file.yaml b/.github/workflows/check-action-file.yaml index 95ad94b..b510deb 100644 --- a/.github/workflows/check-action-file.yaml +++ b/.github/workflows/check-action-file.yaml @@ -19,6 +19,18 @@ on: - 'scripts/select_coverage_artifacts.py' - 'scripts/test_select_coverage_artifacts.py' - 'scripts/test_coverage_native_runtime.py' + - 'scripts/coverage_skip_manifest.py' + - 'scripts/test_coverage_skip_manifest.py' + - 'scripts/test_cs_s3_coverage.py' + - 'scripts/coverage_two_groups.py' + - 'scripts/coverage_two_groups_merge.py' + - 'scripts/coverage_group_collect.py' + - 'scripts/run_coverage_packages.py' + - 'scripts/coverage_timing.py' + - 'scripts/test_coverage_two_groups.py' + - 'scripts/test_coverage_two_groups_merge.py' + - 'scripts/test_coverage_group_collect.py' + - 'scripts/test_run_coverage_packages.py' - 'scripts/test_merge_trigger_tke_subject.py' - 'scripts/compact_coverage_profile.go' - 'scripts/compact_coverage_profile_test.go' @@ -103,6 +115,12 @@ jobs: run: python3 scripts/test_select_coverage_artifacts.py -v - name: Test coverage native runtime paths run: python3 scripts/test_coverage_native_runtime.py -v + - name: Test CS S3 routing and skip comparison contracts + timeout-minutes: 3 + run: python3 -m unittest scripts.test_cs_s3_coverage scripts.test_coverage_skip_manifest scripts.test_summarize_coverage_ut scripts.test_run_coverage_ut -v + - name: Test three-group scheduling and artifact selection + timeout-minutes: 3 + run: PYTHONPATH=scripts python3 -m unittest scripts.test_coverage_two_groups scripts.test_coverage_two_groups_merge scripts.test_coverage_group_collect scripts.test_run_coverage_packages -v - name: Test TKE merge subject identity contract run: python3 scripts/test_merge_trigger_tke_subject.py -v - name: Test SCA license scope selection diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml index 12c2e0d..47eeff3 100644 --- a/.github/workflows/coverage-ut.yaml +++ b/.github/workflows/coverage-ut.yaml @@ -4,7 +4,7 @@ on: workflow_call: inputs: cache_seed: - description: "Opt-in host-ut-v1 cache canary; keep disabled until end-to-end cost is measured" + description: "Compatibility input; CS three-group workflow does not seed external caches" required: false type: boolean default: false @@ -19,31 +19,18 @@ on: value: ${{ jobs.check_coverage_eligibility.outputs.allowed }} coverage_status: description: "Terminal UT coverage producer status" - value: ${{ jobs['ut-coverage-linux-x86'].outputs.status }} + value: ${{ jobs['coverage-final-status'].outputs.status }} secrets: - TOKEN_ACTION: - description: "Token for checkout (e.g. pull from fork/private)" - required: false - TEST_S3FS_ALIYUN: - description: "Object storage configuration for UT coverage" - required: false - TEST_S3FS_QCLOUD: - description: "Object storage configuration for UT coverage" - required: false - S3ENDPOINT: - description: "S3ENDPOINT For Test" + CS_S3_MTY_CONFIG: required: false - S3REGION: - description: "S3REGION For Test" + CS_S3_MNL2_CONFIG: required: false - S3APIKEY: - description: "S3APIKEY For Test" + CS_S3_JED_CONFIG: required: false - S3APISECRET: - description: "S3APISECRET For Test" + CS_S3_RUH_CONFIG: required: false - S3BUCKET: - description: "S3BUCKET For Test" + TOKEN_ACTION: + description: "Token for checkout (e.g. pull from fork/private)" required: false jobs: @@ -97,17 +84,16 @@ jobs: - id: check_coverage_secrets name: Check coverage secrets env: - TEST_S3FS_ALIYUN: ${{ secrets.TEST_S3FS_ALIYUN }} - TEST_S3FS_QCLOUD: ${{ secrets.TEST_S3FS_QCLOUD }} - S3ENDPOINT: ${{ secrets.S3ENDPOINT }} - S3REGION: ${{ secrets.S3REGION }} - S3APIKEY: ${{ secrets.S3APIKEY }} - S3APISECRET: ${{ secrets.S3APISECRET }} - S3BUCKET: ${{ secrets.S3BUCKET }} + CS_S3_MTY_CONFIG: ${{ secrets.CS_S3_MTY_CONFIG }} + CS_S3_MNL2_CONFIG: ${{ secrets.CS_S3_MNL2_CONFIG }} + CS_S3_JED_CONFIG: ${{ secrets.CS_S3_JED_CONFIG }} + CS_S3_RUH_CONFIG: ${{ secrets.CS_S3_RUH_CONFIG }} + IN_ORG: ${{ steps.check_in_org.outputs.in_org }} + SAFE_LABEL: ${{ steps.check_safe_label.outputs.safe_label }} run: | set -euo pipefail missing=() - for name in TEST_S3FS_ALIYUN TEST_S3FS_QCLOUD S3ENDPOINT S3REGION S3APIKEY S3APISECRET S3BUCKET; do + for name in CS_S3_MTY_CONFIG CS_S3_MNL2_CONFIG CS_S3_JED_CONFIG CS_S3_RUH_CONFIG; do if [ -z "${!name:-}" ]; then missing+=("${name}") fi @@ -116,6 +102,10 @@ jobs: echo 'ready=true' >> "$GITHUB_OUTPUT" else echo 'ready=false' >> "$GITHUB_OUTPUT" + if [ "${IN_ORG}" = '1' ] || [ "${SAFE_LABEL}" = '1' ]; then + echo "::error::CS runner S3 configurations must be set before enabling Coverage: ${missing[*]}" + exit 1 + fi echo "::notice::UT coverage skipped because required secrets are unavailable: ${missing[*]}" fi - id: eligibility @@ -134,28 +124,39 @@ jobs: echo '::notice::UT coverage skipped for an untrusted pull request' fi fi - ut-coverage-linux-x86: - name: UT Coverage on Ubuntu/x86 + name: UT Coverage group ${{ matrix.group }} on Ubuntu/x86 + strategy: + fail-fast: false + matrix: + include: + - group: 0 + - group: 1 + - group: 2 outputs: status: ${{ steps.coverage_status.outputs.status }} needs: [check_coverage_eligibility] if: ${{ needs.check_coverage_eligibility.outputs.allowed == 'true' }} environment: ci - # Preserve the Guangzhou network path and capacity used by main's - # external Aliyun/QCloud coverage UT. This is intentionally larger than - # the UT-only minimum while we validate the external-storage timeout fix. - runs-on: amd64-mo-guangzhou-2xlarge16 - # The cache-enabled failure path has 305m of step limits: 20m for the - # two checkouts, 20m cache seeding, 190m coverage, 15m Go/Python setup, - # 5m cache status, and 55m for failure reports, artifacts, final status, - # and failure propagation. Reserve 5m for job overhead and post-action - # cleanup. Success paths skip the failure-only steps. + # Schedule across CS environments; bind S3 to the actual runner below. + runs-on: cs-large-7c14g + # Leave headroom for setup, cache seeding, reports and post-action cleanup. + # Success paths skip the failure-only steps. # The wrapper below enforces the test deadline before this job-level last # resort; the job timeout is never the diagnostic mechanism for a stuck # test. - timeout-minutes: 310 + timeout-minutes: 320 steps: + - name: Install CS runner build dependencies + timeout-minutes: 10 + run: | + set -euo pipefail + if command -v gcc && command -v g++ && command -v make && command -v pkg-config && command -v python; then + echo 'Build dependencies are already available.' + else + sudo apt-get -o Acquire::ForceIPv4=true update + sudo apt-get -o Acquire::ForceIPv4=true install -y build-essential pkg-config python3 python-is-python3 + fi - name: Checkout MatrixOne source # Low-speed limits apply after connection; proxy CONNECT/TLS stalls # can still consume libcurl's 300s connection timeout. The 10m step @@ -171,12 +172,14 @@ jobs: fetch-depth: "3" repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false - name: Set up Go timeout-minutes: 10 uses: matrixorigin/CI/actions/setup-env@main with: setup-java: false go-version-file: "${{ github.workspace }}/go.mod" + cache: false - name: Prepare Python UDF contract tests if: ${{ hashFiles('pkg/udf/python/worker/requirements.txt') != '' }} timeout-minutes: 5 @@ -195,26 +198,16 @@ jobs: sparse-checkout: | scripts/prepare_coverage_cgo.sh scripts/run_coverage_ut.py + scripts/run_coverage_packages.py + scripts/coverage_two_groups.py + scripts/test_run_coverage_packages.py scripts/compact_coverage_profile.go scripts/summarize_coverage_ut.py + scripts/coverage_timing.py + scripts/coverage_skip_manifest.py scripts/summarize_ut_report.py sparse-checkout-cone-mode: false path: .ci-workflow - # Import completed seed entries, not a directory-emptiness heuristic. - # The trusted action records cost/provenance; Go still validates cache keys. - - id: seed_caches - if: ${{ inputs.cache_seed }} - name: Seed Go caches from CI builder image - timeout-minutes: 20 - continue-on-error: true - uses: matrixorigin/CI/actions/seed-go-caches@main - with: - flavor: coverage - - name: Record cache seed outcome - if: ${{ always() && !cancelled() }} - timeout-minutes: 5 - run: | - echo "- Go cache seed outcome: \`${{ steps.seed_caches.outcome }}\`." >> "$GITHUB_STEP_SUMMARY" - id: coverage_ut name: Run coverage unit tests # This step includes bounded package discovery and native preparation @@ -228,7 +221,10 @@ jobs: timeout-minutes: 190 continue-on-error: true env: - GOPROXY: http://goproxy.goproxy.svc.cluster.local,https://goproxy.cn,https://proxy.golang.org + GOPROXY: https://goproxy.cn|https://proxy.golang.org|direct + GOMAXPROCS: "7" + CS_S3_ENV: ${{ startsWith(runner.name, 'cs-mty-') && 'mty' || startsWith(runner.name, 'cs-mnl2-') && 'mnl2' || startsWith(runner.name, 'cs-jed-') && 'jed' || startsWith(runner.name, 'cs-ruh-') && 'ruh' || '' }} + TEST_S3FS_CS: ${{ startsWith(runner.name, 'cs-mty-') && secrets.CS_S3_MTY_CONFIG || startsWith(runner.name, 'cs-mnl2-') && secrets.CS_S3_MNL2_CONFIG || startsWith(runner.name, 'cs-jed-') && secrets.CS_S3_JED_CONFIG || startsWith(runner.name, 'cs-ruh-') && secrets.CS_S3_RUH_CONFIG || '' }} COVERAGE_UT_TEST_TIMEOUT_SECONDS: "3600" # The go test package deadline below is 20m; it must leave time for # the runner's 60m whole-suite deadline and diagnostics. @@ -240,19 +236,23 @@ jobs: # Bound cross-process cluster overlap without serializing all eight # packages on one admission lock. Same-process exclusion is preserved. MO_TEST_CLUSTER_ADMISSION_POOL_SIZE: "2" - COVERAGE_UT_PACKAGE_PARALLELISM: "8" + COVERAGE_UT_PACKAGE_PARALLELISM: "7" COVERAGE_UT_DISCOVERY_TIMEOUT_SECONDS: "900" COVERAGE_UT_CGO_TIMEOUT_SECONDS: "1800" + COVERAGE_GROUP: ${{ matrix.group }} run: | set -euo pipefail cd "$GITHUB_WORKSPACE" - export endpoint='${{ secrets.S3ENDPOINT }}' - export region='${{ secrets.S3REGION }}' - export apikey='${{ secrets.S3APIKEY }}' - export apisecret='${{ secrets.S3APISECRET }}' - export bucket='${{ secrets.S3BUCKET }}' - export TEST_S3FS_ALIYUN='${{ secrets.TEST_S3FS_ALIYUN }}' - export TEST_S3FS_QCLOUD='${{ secrets.TEST_S3FS_QCLOUD }}' + echo "Coverage comparison: GOMAXPROCS=${GOMAXPROCS}; package parallelism=${COVERAGE_UT_PACKAGE_PARALLELISM}" + case "${CS_S3_ENV:-}" in + mty|mnl2|jed|ruh) ;; + *) echo '::error::Unknown CS runner environment; refusing to choose another environment storage.'; exit 1 ;; + esac + test -n "${TEST_S3FS_CS:-}" || { echo '::error::Missing S3 configuration for the selected CS runner.'; exit 1; } + export MO_CI_S3_PREFIX="ci-ut/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/group-${COVERAGE_GROUP}" + unset endpoint region apikey apisecret bucket TEST_S3FS_ALIYUN TEST_S3FS_QCLOUD + echo "- Coverage runner environment: ${CS_S3_ENV}; storage: same-environment CS S3; prefix: ${MO_CI_S3_PREFIX}." >> "$GITHUB_STEP_SUMMARY" + echo '::notice::CS S3 mode: COS and OSS provider integration tests are not executed.' coverage_profile="${RUNNER_TEMP}/ut-coverage.out" coverage_report="${RUNNER_TEMP}/ut-coverage-report.json" coverage_progress="${RUNNER_TEMP}/ut-coverage-progress.txt" @@ -360,7 +360,13 @@ jobs: (( phase_status == 0 )) || exit "${phase_status}" package_count=$(printf '%s\n' "${test_scope}" | awk 'NF { count++ } END { print count + 0 }') instrumented_package_count=$(printf '%s\n' "${cover_pkgs}" | awk -F, '{ print NF }') - echo "coverage_ut package_count=${package_count} package_parallelism=${COVERAGE_UT_PACKAGE_PARALLELISM} instrumented_package_count=${instrumented_package_count}" | tee -a "${phase_timing}" + printf '%s\n' "${test_scope}" > "$RUNNER_TEMP/coverage-full-scope.txt" + python "$GITHUB_WORKSPACE/.ci-workflow/scripts/coverage_two_groups.py" \ + --packages "$RUNNER_TEMP/coverage-full-scope.txt" --group "${COVERAGE_GROUP}" \ + --manifest "$RUNNER_TEMP/coverage-group-plan.json" --selected "$RUNNER_TEMP/coverage-group.txt" + test_scope=$(cat "$RUNNER_TEMP/coverage-group.txt") + selected_count=$(wc -l < "$RUNNER_TEMP/coverage-group.txt" | tr -d ' ') + echo "coverage_ut package_count=${selected_count} full_package_count=${package_count} package_parallelism=${COVERAGE_UT_PACKAGE_PARALLELISM} instrumented_package_count=${instrumented_package_count}" | tee -a "${phase_timing}" phase_start prepare_cgo set +e @@ -374,8 +380,27 @@ jobs: cgo_cflags="-I${GITHUB_WORKSPACE}/cgo -I${thirdparties_install_dir}/include" cgo_ldflags="-L${GITHUB_WORKSPACE}/cgo -lmo -Wl,-rpath,${GITHUB_WORKSPACE}/cgo -L${thirdparties_install_dir}/lib -lusearch_c -Wl,-rpath,${thirdparties_install_dir}/lib -lm" - # Keep the same all-package coverage scope and external storage - # coverage as main. Save the verbose JSON report as an artifact + # Fail early on the real restricted-account SDK path before the suite. + phase_start cs_s3_smoke + set +e + env CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" \ + go test -mod=readonly -count=1 -json -short -tags matrixone_test -timeout=5m \ + ./pkg/fileservice -run '^(TestCSStoragePrefixScopedInitialization|TestCITestS3ErrorRedactsDetails|TestCSStorageArgumentsPreservePrefix|TestCITestS3FixtureSafety|TestValidCITestS3Prefix|TestObjectStorages|TestS3FSFromSpecs)$' \ + > "$RUNNER_TEMP/cs-s3-smoke.json" + smoke_status=$? + set -e + phase_end "${smoke_status}" + if (( smoke_status != 0 )); then + jq -r 'select(.Action == "fail" or (.Action == "output" and (.Output | contains("CS S3"))))' "$RUNNER_TEMP/cs-s3-smoke.json" + exit "${smoke_status}" + fi + for required in 'TestObjectStorages/cs-s3/minio/basic' 'TestS3FSFromSpecs/cs-s3/streaming_write' 'TestCSStoragePrefixScopedInitialization'; do + jq -e --arg test_name "$required" 'select(.Action == "pass" and .Test == $test_name)' "$RUNNER_TEMP/cs-s3-smoke.json" >/dev/null || exit 1 + done + echo '- Restricted-account S3 initialization, basic and streaming smoke passed.' >> "$GITHUB_STEP_SUMMARY" + + # Keep the same all-package scope, using environment-local CS S3. + # Save the verbose JSON report as an artifact # rather than flooding the Actions console; the next step prints a # concise parsed failure summary when this command fails. set +e @@ -437,10 +462,9 @@ jobs: --heartbeat-seconds "${COVERAGE_UT_HEARTBEAT_SECONDS}" \ --console-heartbeat-seconds "${COVERAGE_UT_CONSOLE_HEARTBEAT_SECONDS}" \ -- env CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" \ - go test -mod=readonly -count=1 -json -short -v -tags matrixone_test \ - -timeout=20m \ - -p "${COVERAGE_UT_PACKAGE_PARALLELISM}" \ - -covermode=set -coverprofile="${coverage_profile}" -coverpkg="${cover_pkgs}" \ + python "$GITHUB_WORKSPACE/.ci-workflow/scripts/run_coverage_packages.py" \ + --parallel "${COVERAGE_UT_PACKAGE_PARALLELISM}" \ + --profile "${coverage_profile}" --coverpkg="${cover_pkgs}" \ ${test_scope} & coverage_runner_pid=$! fallback_deadline=$((SECONDS + COVERAGE_UT_FALLBACK_TIMEOUT_SECONDS)) @@ -536,6 +560,14 @@ jobs: fi test -s "${coverage_report}" test -s "${coverage_profile}" + if [[ "${COVERAGE_GROUP}" == '0' ]]; then + for required in 'TestObjectStorages/cs-s3/minio/basic' 'TestS3FSFromSpecs/cs-s3/streaming_write'; do + jq -e --arg test_name "$required" 'select(.Action == "pass" and .Package == "github.com/matrixorigin/matrixone/pkg/fileservice" and .Test == $test_name)' "${coverage_report}" >/dev/null || { + echo "::error::Required CS S3 test did not pass: ${required}" + exit 1 + } + done + fi phase_start compact_profile # -coverpkg repeats instrumented blocks for every test package. The # Go helper ORs exact block hits with memory proportional to unique @@ -646,7 +678,7 @@ jobs: timeout-minutes: 15 uses: matrixorigin/CI/actions/upload-artifact-with-retry@main with: - name: ut-coverage-generation-${{ inputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} + name: ut-coverage-group-${{ matrix.group }}-generation-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/ut-coverage.out if-no-files-found: error retention-days: 7 @@ -722,6 +754,40 @@ jobs: else echo '::warning::failed to render the readable coverage UT report' fi + - name: Summarize package and admission timings + if: ${{ always() && !cancelled() }} + timeout-minutes: 5 + continue-on-error: true + run: | + python "$GITHUB_WORKSPACE/.ci-workflow/scripts/coverage_timing.py" \ + "$RUNNER_TEMP/ut-coverage-report.json" "$RUNNER_TEMP/ut-coverage-timing.json" + - id: skip_manifest + name: Preserve per-test skip evidence on success and failure + if: ${{ always() && !cancelled() }} + timeout-minutes: 5 + env: + ACTUAL_RUNNER: ${{ runner.name }} + STORAGE_MODE: cs-s3 + run: | + set -euo pipefail + python "$GITHUB_WORKSPACE/.ci-workflow/scripts/coverage_skip_manifest.py" \ + --report "$RUNNER_TEMP/ut-coverage-report.json" \ + --status "$RUNNER_TEMP/ut-coverage-status.json" \ + --source-sha "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" \ + --runner "$ACTUAL_RUNNER" --mode "$STORAGE_MODE" \ + --output "$RUNNER_TEMP/ut-coverage-skip-manifest.json" + - name: Upload skip comparison evidence + if: ${{ always() && !cancelled() && steps.skip_manifest.outcome == 'success' }} + timeout-minutes: 5 + uses: actions/upload-artifact@v7 + with: + name: ut-coverage-skips-group-${{ matrix.group }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/ut-coverage-skip-manifest.json + ${{ runner.temp }}/coverage-group-plan.json + ${{ runner.temp }}/coverage-group.txt + if-no-files-found: error + retention-days: 14 - id: upload_coverage_report name: Upload readable coverage UT report if: ${{ always() && !cancelled() && steps.render_coverage_report.outcome == 'success' }} @@ -729,8 +795,11 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v7 with: - name: ut-coverage-report-attempt-${{ github.run_attempt }} - path: ${{ runner.temp }}/ut-coverage-incident.md + name: ut-coverage-report-group-${{ matrix.group }}-attempt-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/ut-coverage-incident.md + ${{ runner.temp }}/ut-coverage.stderr.log + ${{ runner.temp }}/ut-coverage-timing.json if-no-files-found: warn retention-days: 7 - id: upload_coverage_diagnostics @@ -740,9 +809,10 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v7 with: - name: ut-coverage-diagnostics-attempt-${{ github.run_attempt }} + name: ut-coverage-diagnostics-group-${{ matrix.group }}-attempt-${{ github.run_attempt }} path: | ${{ runner.temp }}/ut-coverage-report.json + ${{ runner.temp }}/ut-coverage.stderr.log ${{ runner.temp }}/ut-failure-summary.md ${{ runner.temp }}/ut-report ${{ runner.temp }}/ut-runner-diagnostics.txt @@ -762,3 +832,78 @@ jobs: if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} timeout-minutes: 5 run: exit 1 + + merge-coverage-groups: + needs: [ut-coverage-linux-x86] + if: ${{ needs.ut-coverage-linux-x86.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + token: ${{ secrets.TOKEN_ACTION }} + persist-credentials: false + sparse-checkout: | + scripts/coverage_group_collect.py + scripts/coverage_two_groups_merge.py + scripts/compact_coverage_profile.go + sparse-checkout-cone-mode: false + - uses: actions/setup-go@v5 + with: + go-version: '1.23' + cache: false + - uses: actions/download-artifact@v4 + with: + pattern: ut-coverage-skips-group-*-${{ github.run_id }}-* + path: raw/groups + - uses: actions/download-artifact@v4 + with: + pattern: ut-coverage-group-*-generation-${{ github.run_id }}-* + path: raw/profiles + - name: Validate complete package evidence and merge profiles + env: + EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + test -n "$EXPECTED_HEAD_SHA" + python3 scripts/coverage_group_collect.py --groups raw/groups --profiles raw/profiles --output selected --run-id "$GITHUB_RUN_ID" + python3 scripts/coverage_two_groups_merge.py selected/groups combined-skip-manifest.json --expected-sha "$EXPECTED_HEAD_SHA" + test "$(head -n 1 selected/profiles/0/ut-coverage.out)" = 'mode: set' + test "$(head -n 1 selected/profiles/1/ut-coverage.out)" = 'mode: set' + test "$(head -n 1 selected/profiles/2/ut-coverage.out)" = 'mode: set' + { cat selected/profiles/0/ut-coverage.out; tail -n +2 selected/profiles/1/ut-coverage.out; tail -n +2 selected/profiles/2/ut-coverage.out; } > "$RUNNER_TEMP/ut-coverage.out" + go run scripts/compact_coverage_profile.go "$RUNNER_TEMP/ut-coverage.out" + - name: Upload combined UT coverage for existing consumer + uses: matrixorigin/CI/actions/upload-artifact-with-retry@main + with: + name: ut-coverage-generation-${{ inputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/ut-coverage.out + if-no-files-found: error + retention-days: 7 + - name: Upload combined skip evidence + uses: actions/upload-artifact@v7 + with: + name: ut-coverage-combined-skips-${{ github.run_id }}-${{ github.run_attempt }} + path: combined-skip-manifest.json + if-no-files-found: error + retention-days: 7 + + coverage-final-status: + needs: [check_coverage_eligibility, ut-coverage-linux-x86, merge-coverage-groups] + if: ${{ always() && !cancelled() && needs.check_coverage_eligibility.outputs.allowed == 'true' }} + runs-on: ubuntu-latest + outputs: + status: ${{ steps.result.outputs.status }} + steps: + - id: result + env: + GROUP_RESULT: ${{ needs.ut-coverage-linux-x86.result }} + MERGE_RESULT: ${{ needs.merge-coverage-groups.result }} + run: | + if [ "$GROUP_RESULT" = success ] && [ "$MERGE_RESULT" = success ]; then + echo 'status=passed' >> "$GITHUB_OUTPUT" + else + echo 'status=failed' >> "$GITHUB_OUTPUT" + fi diff --git a/scripts/coverage_group_collect.py b/scripts/coverage_group_collect.py new file mode 100644 index 0000000..170c3f3 --- /dev/null +++ b/scripts/coverage_group_collect.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Select matching latest group evidence and profiles for one Actions run.""" +import argparse +import re +from pathlib import Path + + +def collect(raw_groups, raw_profiles, output, run_id): + if not str(run_id).isdigit(): + raise ValueError('invalid run ID') + root = Path(output) + for group in range(3): + def candidates(directory, prefix): + pattern = re.compile(rf'^{prefix}-{group}-(?:generation-)?{run_id}-([1-9][0-9]*)$') + return {int(match.group(1)): p for p in Path(directory).iterdir() + if p.is_dir() and (match := pattern.fullmatch(p.name))} + evidence = candidates(raw_groups, 'ut-coverage-skips-group') + profiles = candidates(raw_profiles, 'ut-coverage-group') + if not evidence or not profiles or max(evidence) != max(profiles): + raise ValueError(f'group {group}: missing or mismatched latest artifacts') + attempt = max(evidence) + group_out = root / 'groups' / str(group) + profile_out = root / 'profiles' / str(group) + group_out.mkdir(parents=True) + profile_out.mkdir(parents=True) + for filename in ('coverage-group-plan.json', 'ut-coverage-skip-manifest.json'): + source = evidence[attempt] / filename + if not source.is_file() or source.is_symlink(): + raise ValueError(f'group {group}: missing evidence') + (group_out / filename).symlink_to(source.resolve()) + profile = profiles[attempt] / 'ut-coverage.out' + if not profile.is_file() or profile.is_symlink(): + raise ValueError(f'group {group}: missing profile') + (profile_out / 'ut-coverage.out').symlink_to(profile.resolve()) + print(f'group {group}: selected attempt {attempt}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--groups', required=True) + parser.add_argument('--profiles', required=True) + parser.add_argument('--output', required=True) + parser.add_argument('--run-id', required=True) + args = parser.parse_args() + collect(args.groups, args.profiles, args.output, args.run_id) diff --git a/scripts/coverage_skip_manifest.py b/scripts/coverage_skip_manifest.py new file mode 100644 index 0000000..aff44cf --- /dev/null +++ b/scripts/coverage_skip_manifest.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Keep small, credential-redacted Coverage skip evidence on success and failure.""" + +import argparse +from collections import Counter, deque +import json +from pathlib import Path +import re + + +def redact(text): + text = re.sub(r"https?://\S+", "", text) + text = re.sub(r"\S+@tcp\([^)]*\)\S*", "", text) + text = re.sub(r"(?i)((?:password|secret|token|access[_-]?key|api[_-]?key|key-id|key-secret)\s*[:=]\s*)[^\s,;]+", r"\1", text) + return text[:800] + + +def collect(report, status, source_sha, runner, mode): + counts = Counter() + started_packages = set() + notes = {} + skipped, packages, failed, timeouts = [], [], [], [] + malformed = 0 + with report.open(errors="replace") as stream: + for line in stream: + try: + event = json.loads(line) + except ValueError: + malformed += 1 + continue + if not isinstance(event, dict) or not isinstance(event.get("Action"), str): + malformed += 1 + continue + action = event["Action"] + counts[action] += 1 + if action == "start": + started_packages.add(event.get("Package", "")) + key = (event.get("Package", ""), event.get("Test", "")) + item = {"package": key[0], "test": key[1]} + if action == "output": + message = event.get("Output", "").strip() + # Keep only Go test messages, not arbitrary application logs. + if re.match(r"[^\s]+\.go:\d+:", message): + notes.setdefault(key, deque(maxlen=8)).append(redact(message)) + if re.search(r"panic: test timed out after|^test timed out after", message): + timeouts.append(dict(item, reason=redact(message))) + elif action == "skip": + item["reasons"] = list(notes.pop(key, [])) or [ + "skip without message" if key[1] else "package has no test files" + ] + (skipped if key[1] else packages).append(item) + elif action in ("pass", "fail"): + notes.pop(key, None) + if action == "fail": + failed.append(item) + return { + "schema": 1, "source_sha": source_sha, "runner": runner, "storage_mode": mode, + "producer_status": status.get("result", "unknown"), + "producer_reason": status.get("reason", "unknown"), + "elapsed_seconds": status.get("elapsed_seconds"), + "malformed_events": malformed, "event_counts": dict(counts), + "started_packages": sorted(started_packages), + "skipped_tests": skipped, "packages_without_tests": packages, + "failed": failed, "package_timeouts": timeouts, + "note": "Real COS/OSS integration is intentionally outside CS UT scope. Absent dynamic storage cases do not appear as skip events.", + } + + +def compare(baseline, candidate): + def identities(items): + return {(x["package"], x["test"]) for x in items} + + old = identities(baseline["skipped_tests"]) + new = identities(candidate["skipped_tests"]) + added = sorted(new - old) + reasons_changed = [] + old_reasons = {(x["package"], x["test"]): x["reasons"] for x in baseline["skipped_tests"]} + for item in candidate["skipped_tests"]: + key = (item["package"], item["test"]) + if key in old_reasons and old_reasons[key] != item["reasons"]: + reasons_changed.append(item) + comparable = baseline["source_sha"] == candidate["source_sha"] and bool(baseline["source_sha"]) + same_packages = bool(baseline["started_packages"]) and baseline["started_packages"] == candidate["started_packages"] + healthy = all(x["producer_status"] == "passed" and not x["malformed_events"] + and not x["failed"] and not x["package_timeouts"] for x in (baseline, candidate)) + return { + "same_source": comparable, "same_package_scope": same_packages, "both_producers_passed": healthy, + "new_skips": [{"package": p, "test": t} for p, t in added], + "removed_skips": [{"package": p, "test": t} for p, t in sorted(old - new)], + "changed_skip_reasons": reasons_changed, + "baseline_skip_tests": len(old), "candidate_skip_tests": len(new), + "accepted": comparable and same_packages and healthy and not added, + "scope": "Existing skipped identities are accepted; real COS/OSS coverage is not required. Changed reasons are retained for review, not treated as new skips.", + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--report", type=Path) + parser.add_argument("--status", type=Path) + parser.add_argument("--source-sha") + parser.add_argument("--runner", default="unknown") + parser.add_argument("--mode", default="unknown") + parser.add_argument("--baseline", type=Path) + parser.add_argument("--candidate", type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + if args.baseline and args.candidate: + result = compare(json.loads(args.baseline.read_text()), json.loads(args.candidate.read_text())) + exit_code = 0 if result["accepted"] else 1 + elif args.report and args.status and args.source_sha: + result = collect(args.report, json.loads(args.status.read_text()), args.source_sha, args.runner, args.mode) + exit_code = 0 + else: + parser.error("provide report/status/source-sha or baseline/candidate") + args.output.write_text(json.dumps(result, indent=2) + "\n") + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/coverage_timing.py b/scripts/coverage_timing.py new file mode 100644 index 0000000..67ec44f --- /dev/null +++ b/scripts/coverage_timing.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Extract per-package and embedded-cluster admission timings from Go JSON.""" +import argparse +import json +import re +from collections import defaultdict +from pathlib import Path + +UNIT = {'ns': 1e-9, 'µs': 1e-6, 'us': 1e-6, 'ms': 1e-3, 's': 1, 'm': 60, 'h': 3600} +PART = re.compile(r'(\d+(?:\.\d+)?)(ns|µs|us|ms|s|m|h)') + + +def seconds(value): + parts = list(PART.finditer(value)) + if not parts or ''.join(m.group(0) for m in parts) != value: + raise ValueError('unrecognized Go duration: ' + value) + return sum(float(m.group(1)) * UNIT[m.group(2)] for m in parts) + + +def summarize(path): + packages, waits = {}, defaultdict(list) + parse_errors = 0 + with path.open(encoding='utf-8') as source: + for line in source: + try: + event = json.loads(line) + except (ValueError, UnicodeError): + parse_errors += 1 + continue + package = event.get('Package') + if not package: + continue + if event.get('Action') in ('pass', 'fail', 'skip') and not event.get('Test'): + packages[package] = {'action': event['Action'], 'elapsed_seconds': event.get('Elapsed')} + output = event.get('Output', '') + if 'MO_UT_SETUP' in output and 'phase=admission-acquire' in output: + found = re.search(r'(?:^|\s)duration=([^\s]+)', output) + if found: + try: + waits[package].append({'test': event.get('Test'), 'seconds': seconds(found.group(1))}) + except ValueError: + parse_errors += 1 + return {'packages': packages, 'admission_observations': dict(waits), 'parse_errors': parse_errors, + 'limitations': 'Admission values cover only emitted embedded-cluster setup records; package times overlap.'} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('report', type=Path) + parser.add_argument('output', type=Path) + args = parser.parse_args() + with args.output.open('x', encoding='utf-8') as out: + json.dump(summarize(args.report), out, ensure_ascii=False, indent=2) + out.write('\n') diff --git a/scripts/coverage_two_groups.py b/scripts/coverage_two_groups.py new file mode 100644 index 0000000..01b3e33 --- /dev/null +++ b/scripts/coverage_two_groups.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Deterministically partition a complete Go package list into three groups. + +Weights are approximate elapsed seconds from one MNL2 Coverage run; they are +only scheduling hints, never an allowlist. Unknown packages are still tested. +""" +import argparse +import hashlib +import json +from pathlib import Path + +ROOT = 'github.com/matrixorigin/matrixone/' +WEIGHTS = { + 'pkg/tests/sqlintegration': 879, 'pkg/tests/service': 638, + 'pkg/tests/issues/isolated': 633, 'pkg/embed': 511, + 'pkg/tests/issues': 474, 'pkg/sql/plan': 219, + 'pkg/tests/sqlintegration/multicn': 208, 'pkg/vm/engine/test': 189, + 'pkg/tests/arrowload': 142, 'pkg/logservice': 127, + 'pkg/tests/upgrade': 117, 'pkg/tests/partition': 101, + 'pkg/tests/dml': 94, 'pkg/vm/engine/tae/db/test': 83, + 'optools': 82, 'pkg/shardservice': 57, + 'pkg/vm/engine/disttae': 54, 'pkg/tests/shard': 54, + 'pkg/lockservice': 50, 'pkg/frontend': 46, +} +FILE_SERVICE = ROOT + 'pkg/fileservice' +EMBED = ROOT + 'pkg/embed' + + +def plan(packages): + if not packages or len(set(packages)) != len(packages): + raise ValueError('empty or duplicate package scope') + if FILE_SERVICE not in packages or EMBED not in packages: + raise ValueError('required S3 or embed test package is missing') + # Seed heavy packages into different groups; shared-pool jobs can land in + # any environment, so no group is assumed to use a particular runner. + groups = [[FILE_SERVICE], [], [EMBED]] + weights = [5, 0, WEIGHTS['pkg/embed']] + for package in sorted(set(packages) - {FILE_SERVICE, EMBED}, + key=lambda p: (-WEIGHTS.get(p.removeprefix(ROOT), 5), p)): + index = min(range(3), key=lambda i: (weights[i], i)) + groups[index].append(package) + weights[index] += WEIGHTS.get(package.removeprefix(ROOT), 5) + groups = [sorted(group) for group in groups] + if not all(groups) or sum(len(set(g)) for g in groups) != len(set().union(*map(set, groups))) or set().union(*map(set, groups)) != set(packages): + raise ValueError('invalid partition') + scope = sorted(packages) + digest = hashlib.sha256(('\n'.join(scope) + '\n').encode()).hexdigest() + return {'schema': 1, 'scope_sha256': digest, 'scope_count': len(scope), + 'groups': groups, 'estimated_weight': weights} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--packages', required=True, type=Path) + parser.add_argument('--group', required=True, type=int, choices=(0, 1, 2)) + parser.add_argument('--manifest', required=True, type=Path) + parser.add_argument('--selected', required=True, type=Path) + args = parser.parse_args() + result = plan(args.packages.read_text().splitlines()) + result['selected_group'] = args.group + args.manifest.write_text(json.dumps(result, sort_keys=True) + '\n') + args.selected.write_text('\n'.join(result['groups'][args.group]) + '\n') + print(f"group={args.group} selected={len(result['groups'][args.group])} " + f"scope={result['scope_count']} scope_sha256={result['scope_sha256']}") + + +if __name__ == '__main__': + main() diff --git a/scripts/coverage_two_groups_merge.py b/scripts/coverage_two_groups_merge.py new file mode 100644 index 0000000..54d2580 --- /dev/null +++ b/scripts/coverage_two_groups_merge.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Validate complete group evidence before merging coverage profiles.""" +import argparse +import json +from pathlib import Path + + +def validate(root, expected_sha=None): + plans = [json.loads((root / str(i) / 'coverage-group-plan.json').read_text()) for i in range(3)] + results = [json.loads((root / str(i) / 'ut-coverage-skip-manifest.json').read_text()) for i in range(3)] + for i in range(3): + p, r = plans[i], results[i] + if p['selected_group'] != i or r['producer_status'] != 'passed': + raise ValueError('group identity or producer status mismatch') + if r['malformed_events'] or r['failed'] or r['package_timeouts']: + raise ValueError('group reported failed tests, malformed events, or timeout') + if set(r['started_packages']) != set(p['groups'][i]): + raise ValueError('actual package scope does not match group plan') + if any(plans[0][key] != plan[key] for plan in plans[1:] for key in ('scope_sha256', 'scope_count', 'groups')): + raise ValueError('group plans differ') + groups = list(map(set, plans[0]['groups'])) + if len(groups) != 3 or not all(groups) or sum(map(len, groups)) != len(set().union(*groups)) or len(set().union(*groups)) != plans[0]['scope_count']: + raise ValueError('incomplete or overlapping group scope') + if any(results[0]['source_sha'] != result['source_sha'] for result in results[1:]): + raise ValueError('source mismatch') + if expected_sha and results[0]['source_sha'] != expected_sha: + raise ValueError('source does not match the PR head') + return results + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('root', type=Path) + parser.add_argument('output', type=Path) + parser.add_argument('--expected-sha', required=True) + args = parser.parse_args() + results = validate(args.root, args.expected_sha) + result = dict(results[0]) + for key in ('started_packages', 'skipped_tests', 'packages_without_tests', 'failed', 'package_timeouts'): + result[key] = sum((item[key] for item in results), []) + result['started_packages'] = sorted(result['started_packages']) + result['runner'] = 'three isolated runner jobs' + args.output.write_text(json.dumps(result, indent=2) + '\n') + print(f"Validated {len(result['started_packages'])} packages across all groups") + + +if __name__ == '__main__': + main() diff --git a/scripts/run_coverage_packages.py b/scripts/run_coverage_packages.py new file mode 100644 index 0000000..0290dd4 --- /dev/null +++ b/scripts/run_coverage_packages.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Run all selected coverage packages in one bounded parallel group. + +All selected packages execute once with the same instrumentation and deadlines. +The outer run_coverage_ut.py retains the whole-suite deadline/process group. +""" + +import argparse +from pathlib import Path +import shutil +import subprocess +import tempfile + + +EMBED = "github.com/matrixorigin/matrixone/pkg/embed" + + +def groups(packages): + if not packages or len(packages) != len(set(packages)): + raise ValueError("coverage package scope must be nonempty and unique") + return [packages] + + +def run(packages, profile, coverpkg, parallel, execute=subprocess.run): + plan = groups(packages) + if profile.exists(): + raise ValueError("refusing to overwrite an existing coverage profile") + status = 0 + # go test writes -coverpkg dependency warnings to stderr. The outer + # supervisor merges stderr into its JSON stream, so retain compiler output + # separately instead of corrupting the evidence consumed by S3/skip gates. + # A nonzero go exit still fails the run; no warning/error text is discarded. + with tempfile.TemporaryDirectory(prefix="coverage-groups-", dir=profile.parent) as directory, \ + profile.with_suffix(".stderr.log").open("xb") as diagnostics: + parts = [] + for index, group in enumerate(plan): + part = Path(directory) / f"{index}.out" + command = ["go", "test", "-mod=readonly", "-count=1", "-json", "-short", "-v", + "-tags", "matrixone_test", "-timeout=20m", "-p", str(parallel), + "-covermode=set", f"-coverprofile={part}", f"-coverpkg={coverpkg}", *group] + # Inherit the outer process group so its bounded timeout also + # terminates this scheduler, go test and all test descendants. + diagnostics.write(f"group={index} package_count={len(group)}\n".encode()) + diagnostics.flush() + result = execute(command, stderr=diagnostics) + if result.returncode: + status = 1 + parts.append(part) + with profile.open("xb") as output: + output.write(b"mode: set\n") + for part in parts: + if not part.is_file(): + status = 1 + continue + with part.open("rb") as source: + if source.readline() != b"mode: set\n": + status = 1 + continue + shutil.copyfileobj(source, output) + # The existing compact_coverage_profile helper ORs duplicate set-mode + # blocks after this returns. No source/package is dropped here. + return status + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", required=True, type=Path) + parser.add_argument("--coverpkg", required=True) + parser.add_argument("--parallel", required=True, type=int) + parser.add_argument("packages", nargs="+") + args = parser.parse_args() + if args.parallel < 1: + parser.error("parallelism must be positive") + return run(args.packages, args.profile, args.coverpkg, args.parallel) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/summarize_coverage_ut.py b/scripts/summarize_coverage_ut.py index 5ac3b36..b1c4c9f 100644 --- a/scripts/summarize_coverage_ut.py +++ b/scripts/summarize_coverage_ut.py @@ -266,7 +266,7 @@ def render(args: argparse.Namespace) -> str: "", "## Raw evidence", "", - "The full JSON stream and runner diagnostics remain available in the uploaded artifact.", + "Per-test skip evidence is uploaded on success and failure. The full JSON stream and runner diagnostics are uploaded only on failure.", ] ) return cap_markdown("\n".join(lines) + "\n") diff --git a/scripts/test_coverage_group_collect.py b/scripts/test_coverage_group_collect.py new file mode 100644 index 0000000..3e619ee --- /dev/null +++ b/scripts/test_coverage_group_collect.py @@ -0,0 +1,40 @@ +import tempfile +import unittest +from pathlib import Path + +from coverage_group_collect import collect + + +class CollectTest(unittest.TestCase): + def test_latest_matching_attempts_per_group(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + groups, profiles = root / 'groups', root / 'profiles' + groups.mkdir() + profiles.mkdir() + for group, attempt in [(0, 1), (1, 2), (2, 1)]: + evidence = groups / f'ut-coverage-skips-group-{group}-123-{attempt}' + profile = profiles / f'ut-coverage-group-{group}-generation-123-{attempt}' + evidence.mkdir() + profile.mkdir() + for name in ('coverage-group-plan.json', 'ut-coverage-skip-manifest.json'): + (evidence / name).write_text('{}') + (profile / 'ut-coverage.out').write_text('mode: set\n') + collect(groups, profiles, root / 'selected', '123') + self.assertTrue((root / 'selected/groups/1/coverage-group-plan.json').is_file()) + self.assertTrue((root / 'selected/profiles/2/ut-coverage.out').is_file()) + + def test_refuses_mismatched_or_missing_latest_attempt(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + groups, profiles = root / 'groups', root / 'profiles' + groups.mkdir() + profiles.mkdir() + (groups / 'ut-coverage-skips-group-0-123-2').mkdir() + (profiles / 'ut-coverage-group-0-generation-123-1').mkdir() + with self.assertRaises(ValueError): + collect(groups, profiles, root / 'selected', '123') + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_coverage_skip_manifest.py b/scripts/test_coverage_skip_manifest.py new file mode 100644 index 0000000..e1d0d73 --- /dev/null +++ b/scripts/test_coverage_skip_manifest.py @@ -0,0 +1,68 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.coverage_skip_manifest import collect, compare, redact + + +class CoverageSkipManifestTest(unittest.TestCase): + def manifest(self, events, sha="same", status="passed"): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "events.json" + path.write_text("\n".join(json.dumps(e) for e in [{"Action": "start", "Package": "p"}] + events) + "\n") + return collect(path, {"result": status}, sha, "runner", "test") + + def test_distinguishes_packages_and_tests_and_preserves_reasons(self): + result = self.manifest([ + {"Action": "output", "Package": "p", "Test": "T", "Output": "a_test.go:10: missing TEST_CONFIG"}, + {"Action": "skip", "Package": "p", "Test": "T"}, + {"Action": "skip", "Package": "empty"}, + ]) + self.assertEqual(result["event_counts"]["skip"], 2) + self.assertEqual(len(result["packages_without_tests"]), 1) + self.assertEqual(result["skipped_tests"][0]["reasons"], ["a_test.go:10: missing TEST_CONFIG"]) + + def test_original_skips_accepted_but_new_skips_rejected(self): + old = self.manifest([{"Action": "skip", "Package": "p", "Test": "T"}]) + self.assertTrue(compare(old, old)["accepted"]) + new = self.manifest([{"Action": "skip", "Package": "p", "Test": "New"}]) + self.assertFalse(compare(old, new)["accepted"]) + + def test_equal_counts_do_not_hide_different_tests(self): + old = self.manifest([{"Action": "skip", "Package": "p", "Test": "A"}]) + new = self.manifest([{"Action": "skip", "Package": "p", "Test": "B"}]) + self.assertEqual(len(compare(old, new)["new_skips"]), 1) + + def test_different_source_or_failed_producer_not_accepted(self): + old = self.manifest([]) + for new in (self.manifest([], sha="other"), self.manifest([], status="timeout")): + self.assertFalse(compare(old, new)["accepted"]) + + def test_missing_package_or_empty_execution_not_accepted(self): + old = self.manifest([]) + new = self.manifest([]) + new["started_packages"] = [] + self.assertFalse(compare(old, new)["accepted"]) + self.assertFalse(compare(new, new)["accepted"]) + + def test_failure_timeout_and_malformed_evidence_not_accepted(self): + old = self.manifest([]) + new = self.manifest([ + {"Action": "output", "Package": "p", "Output": "panic: test timed out after 20m"}, + {"Action": "fail", "Package": "p"}, + ]) + self.assertFalse(compare(old, new)["accepted"]) + self.assertEqual(len(new["package_timeouts"]), 1) + old["malformed_events"] = 1 + self.assertFalse(compare(old, old)["accepted"]) + + def test_redacts_connections_and_credentials(self): + value = redact("test.go:1: https://user:password@host/path key-secret=opaque dump:password@tcp(host:6001)/") + self.assertNotIn("password", value) + self.assertNotIn("opaque", value) + self.assertNotIn("host", value) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_coverage_two_groups.py b/scripts/test_coverage_two_groups.py new file mode 100644 index 0000000..13beb3f --- /dev/null +++ b/scripts/test_coverage_two_groups.py @@ -0,0 +1,30 @@ +import unittest +from scripts.coverage_two_groups import EMBED, FILE_SERVICE, ROOT, plan + + +class CoverageTwoGroupsTest(unittest.TestCase): + def test_deterministic_complete_and_disjoint(self): + packages = [FILE_SERVICE, ROOT+'pkg/embed', ROOT+'pkg/tests/sqlintegration', ROOT+'pkg/unknown', ROOT+'pkg/a'] + a, b = plan(packages), plan(list(reversed(packages))) + self.assertEqual(a, b) + self.assertEqual(set().union(*map(set, a['groups'])), set(packages)) + self.assertEqual(sum(map(len, a['groups'])), len(packages)) + self.assertIn(FILE_SERVICE, a['groups'][0]) + self.assertIn(EMBED, a['groups'][2]) + + def test_rejects_missing_and_duplicate(self): + with self.assertRaises(ValueError): + plan([ROOT+'pkg/embed']) + with self.assertRaises(ValueError): + plan([FILE_SERVICE, FILE_SERVICE]) + + def test_service_partition_preserves_complete_scope(self): + packages = [FILE_SERVICE, EMBED, ROOT+'pkg/tests/service', ROOT+'pkg/tests/sqlintegration', ROOT+'pkg/tests/issues/isolated', ROOT+'pkg/a'] + result = plan(packages) + self.assertEqual(set().union(*map(set, result['groups'])), set(packages)) + self.assertEqual(sum(map(len, result['groups'])), len(packages)) + self.assertEqual(sum(ROOT+'pkg/tests/service' in g for g in result['groups']), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_coverage_two_groups_merge.py b/scripts/test_coverage_two_groups_merge.py new file mode 100644 index 0000000..e62651b --- /dev/null +++ b/scripts/test_coverage_two_groups_merge.py @@ -0,0 +1,31 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from coverage_two_groups_merge import validate + + +class MergeEvidenceTest(unittest.TestCase): + def test_validation_requires_complete_healthy_disjoint_results(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for group in range(3): + sub = root / str(group) + sub.mkdir() + plan = {'selected_group': group, 'scope_sha256': 'same', 'scope_count': 3, + 'groups': [['pkg/a'], ['pkg/b'], ['pkg/c']]} + report = {'producer_status': 'passed', 'malformed_events': 0, 'failed': [], + 'package_timeouts': [], 'started_packages': [plan['groups'][group][0]], + 'source_sha': 'same'} + (sub / 'coverage-group-plan.json').write_text(json.dumps(plan)) + (sub / 'ut-coverage-skip-manifest.json').write_text(json.dumps(report)) + validate(root) + report['started_packages'] = [] + (root / '1' / 'ut-coverage-skip-manifest.json').write_text(json.dumps(report)) + with self.assertRaises(ValueError): + validate(root) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_cs_s3_coverage.py b/scripts/test_cs_s3_coverage.py new file mode 100644 index 0000000..484b342 --- /dev/null +++ b/scripts/test_cs_s3_coverage.py @@ -0,0 +1,122 @@ +"""Production CS Coverage contracts; no network or real credentials.""" + +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import textwrap +import unittest + + +WORKFLOW = (Path(__file__).parents[1] / ".github/workflows/coverage-ut.yaml").read_text() +STACKS = ("mty", "mnl2", "jed", "ruh") +CONFIGS = tuple(f"CS_S3_{stack.upper()}_CONFIG" for stack in STACKS) + + +def step_script(step): + block = re.split(r"\n(?: - | [\w-]+:)", + WORKFLOW.split(f" - id: {step}\n", 1)[1], maxsplit=1)[0] + return textwrap.dedent(block.split(" run: |\n", 1)[1]) + + +class CSS3CoverageTest(unittest.TestCase): + def gate(self, configs, *, member="1", label="0"): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + env = dict(PATH=os.environ["PATH"], GITHUB_OUTPUT=str(output), IN_ORG=member, SAFE_LABEL=label) + env.update({name: "opaque-test-value" for name in configs}) + result = subprocess.run(["bash", "-c", step_script("check_coverage_secrets")], + env=env, capture_output=True, text=True) + self.assertNotIn("opaque-test-value", result.stdout + result.stderr) + return result.returncode, output.read_text() + + def test_all_environments_required(self): + self.assertEqual(self.gate(CONFIGS), (0, "ready=true\n")) + for missing in CONFIGS: + with self.subTest(missing=missing): + code, output = self.gate([name for name in CONFIGS if name != missing]) + self.assertNotEqual(code, 0) + self.assertEqual(output, "ready=false\n") + self.assertNotEqual(self.gate([], member="0", label="1")[0], 0) + self.assertEqual(self.gate([], member="0"), (0, "ready=false\n")) + + def test_normal_pr_trust_gate_preserved(self): + for member, label, ready, allowed in (("0", "0", "true", False), ("1", "0", "true", True), + ("0", "1", "true", True), ("1", "0", "false", False)): + with self.subTest(member=member, label=label, ready=ready), tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + result = subprocess.run(["bash", "-c", step_script("eligibility")], capture_output=True, + env=dict(PATH=os.environ["PATH"], GITHUB_OUTPUT=str(output), + IN_ORG=member, SAFE_LABEL=label, SECRETS_READY=ready)) + self.assertEqual(result.returncode, 0) + self.assertEqual(output.read_text(), f"allowed={str(allowed).lower()}\n") + + def test_shared_pool_and_pr_source_without_experiment_inputs(self): + self.assertIn("runs-on: cs-large-7c14g", WORKFLOW) + for field, value in (("PR_AUTHOR", "user.login"), ("PR_URL", "url"), + ("repository", "head.repo.full_name"), ("ref", "head.sha")): + self.assertIn(field + ": ${{ github.event.pull_request." + value + " }}", WORKFLOW) + self.assertNotRegex(WORKFLOW, r"inputs\.(canary_\w+|runner_label)|weilu-runner-test|pulls/325\d+|minio-source") + self.assertNotRegex(WORKFLOW, r"secrets\.(TEST_S3FS_(ALIYUN|QCLOUD)|S3\w+)") + self.assertIn('COVERAGE_UT_PACKAGE_PARALLELISM: "7"', WORKFLOW) + self.assertIn('COVERAGE_GROUP: ${{ matrix.group }}', WORKFLOW) + + def test_runner_selects_only_its_own_storage(self): + for stack in (*STACKS, "unknown"): + for missing in (False, True): + for field in ("CS_S3_ENV", "TEST_S3FS_CS"): + with self.subTest(stack=stack, missing=missing, field=field): + expression = next(line.split("${{", 1)[1].split("}}", 1)[0] + for line in WORKFLOW.splitlines() if f"{field}: ${{{{" in line) + expression = re.sub(r"startsWith\(runner.name, '([^']+)'\)", + lambda m: str(f"cs-{stack}-large-7c14g-runner".startswith(m[1])), expression) + for env in STACKS: + value = "" if missing and env == stack else env + expression = expression.replace(f"secrets.CS_S3_{env.upper()}_CONFIG", repr(value)) + expression = expression.replace("&&", " and ").replace("||", " or ") + expected = "" if stack == "unknown" or (missing and field == "TEST_S3FS_CS") else stack + self.assertEqual(eval(expression, {"__builtins__": {}}), expected) + + def test_runtime_rejects_missing_config_and_isolates_attempts(self): + script = step_script("coverage_ut").split('coverage_profile="', 1)[0] + for stack, config, ok in [(s, "opaque-test-value", True) for s in STACKS] + [("unknown", "value", False), ("jed", "", False)]: + for attempt in ("1", "2"): + with self.subTest(stack=stack, config=bool(config), attempt=attempt), tempfile.TemporaryDirectory() as directory: + summary = Path(directory) / "summary" + env = dict(PATH=os.environ["PATH"], GITHUB_WORKSPACE=directory, GITHUB_STEP_SUMMARY=str(summary), + GITHUB_RUN_ID="123", GITHUB_RUN_ATTEMPT=attempt, CS_S3_ENV=stack, TEST_S3FS_CS=config, + COVERAGE_GROUP="0", GOMAXPROCS="7", COVERAGE_UT_PACKAGE_PARALLELISM="7", + endpoint="legacy", TEST_S3FS_ALIYUN="legacy", TEST_S3FS_QCLOUD="legacy") + result = subprocess.run(["bash", "-c", script + '\ntest -z "${endpoint+x}${TEST_S3FS_ALIYUN+x}${TEST_S3FS_QCLOUD+x}"'], + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode == 0, ok) + if ok: + self.assertIn(f"ci-ut/123-{attempt}", summary.read_text()) + self.assertNotIn(config, summary.read_text() + result.stdout + result.stderr) + + def test_required_s3_tests_cannot_be_missing_skipped_or_failed(self): + script = step_script("coverage_ut") + script = 'if [[ "${COVERAGE_GROUP}" == \'0\' ]]; then' + script.split('if [[ "${COVERAGE_GROUP}" == \'0\' ]]; then', 1)[1].split("phase_start compact_profile", 1)[0] + names = ("TestObjectStorages/cs-s3/minio/basic", "TestS3FSFromSpecs/cs-s3/streaming_write") + for action in ("pass", "skip", "fail", "missing"): + with self.subTest(action=action), tempfile.TemporaryDirectory() as directory: + report = Path(directory) / "report.json" + report.write_text("\n".join(json.dumps(dict(Action=action if i else "pass", Test=name, + Package="github.com/matrixorigin/matrixone/pkg/fileservice")) + for i, name in enumerate(names) if not (i and action == "missing"))) + result = subprocess.run(["bash", "-c", script], capture_output=True, + env=dict(PATH=os.environ["PATH"], coverage_report=str(report), COVERAGE_GROUP="0")) + self.assertEqual(result.returncode == 0, action == "pass") + + def test_skip_manifest_uploaded_for_success_and_failure(self): + block = WORKFLOW.split(" - id: skip_manifest\n", 1)[1].split(" - id: upload_coverage_report\n", 1)[0] + self.assertIn("always() && !cancelled()", block) + self.assertNotIn("outcome == 'failure'", block) + self.assertIn("coverage_skip_manifest.py", block) + self.assertIn("ut-coverage-skip-manifest.json", block) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_run_coverage_packages.py b/scripts/test_run_coverage_packages.py new file mode 100644 index 0000000..d40810c --- /dev/null +++ b/scripts/test_run_coverage_packages.py @@ -0,0 +1,110 @@ +import json +import os +import pathlib +import subprocess +import sys +import tempfile +import unittest + +from scripts.run_coverage_packages import EMBED, groups, run + + +class CoveragePackageSchedulingTest(unittest.TestCase): + def test_partition_preserves_every_package_once(self): + packages = ["example/a", EMBED, "example/b"] + plan = groups(packages) + self.assertEqual(plan, [packages]) + self.assertCountEqual([p for group in plan for p in group], packages) + self.assertEqual(groups(["example/a"]), [["example/a"]]) + for invalid in ([], [EMBED, EMBED]): + with self.assertRaises(ValueError): + groups(invalid) + + def test_group_preserves_all_packages_and_failure(self): + for fail_first in (False, True): + with self.subTest(fail_first=fail_first), tempfile.TemporaryDirectory() as directory: + profile = pathlib.Path(directory) / "combined.out" + commands = [] + + def execute(command, *, stderr): + commands.append(command) + stderr.write(b"warning: no packages being tested depend on example/unrelated\n") + part = pathlib.Path(next(v.split("=", 1)[1] for v in command if v.startswith("-coverprofile="))) + part.write_text(f"mode: set\nexample/file.go:1.1,2.1 1 {len(commands)-1}\n") + return subprocess.CompletedProcess(command, 1 if fail_first and len(commands) == 1 else 0) + + status = run([EMBED, "example/a", "example/b"], profile, "example/instrumented", 8, execute) + self.assertEqual(status, int(fail_first)) + self.assertEqual(len(commands), 1) + self.assertEqual(commands[0][-3:], [EMBED, "example/a", "example/b"]) + for command in commands: + self.assertIn("-timeout=20m", command) + self.assertIn("-coverpkg=example/instrumented", command) + self.assertEqual(command[command.index("-p")+1], "8") + self.assertEqual(profile.read_text().count("mode: set"), 1) + self.assertEqual(len(profile.read_text().splitlines()), 2) + diagnostics = profile.with_suffix(".stderr.log").read_text() + self.assertEqual(diagnostics.count("warning: no packages"), 1) + self.assertNotIn("warning:", profile.read_text()) + + def test_missing_profile_fails(self): + with tempfile.TemporaryDirectory() as directory: + result = run([EMBED], pathlib.Path(directory)/"result", "scope", 8, + lambda command, **kwargs: subprocess.CompletedProcess(command, 0)) + self.assertEqual(result, 1) + + def test_existing_profile_is_not_overwritten(self): + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory)/"result" + path.write_text("sentinel") + with self.assertRaises(ValueError): + run([EMBED], path, "scope", 8) + self.assertEqual(path.read_text(), "sentinel") + + def test_existing_diagnostics_are_not_overwritten(self): + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory)/"result.out" + diagnostics = path.with_suffix(".stderr.log") + diagnostics.write_text("existing diagnostic evidence") + with self.assertRaises(FileExistsError): + run([EMBED], path, "scope", 8) + self.assertEqual(diagnostics.read_text(), "existing diagnostic evidence") + + def test_compiler_warning_does_not_corrupt_supervised_json_stream(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + fake_go = root/"go" + fake_go.write_text( + f"#!{sys.executable}\n" + "import json, pathlib, sys\n" + "profile = next(x.split('=',1)[1] for x in sys.argv if x.startswith('-coverprofile='))\n" + "pathlib.Path(profile).write_text('mode: set\\nexample/a.go:1.1,2.1 1 1\\n')\n" + "print('warning: no packages being tested depend on example/a', file=sys.stderr)\n" + "print(json.dumps(dict(Action='pass', Package=sys.argv[-1])))\n" + ) + fake_go.chmod(0o700) + profile = root/"result.out" + report = root/"report.json" + status = root/"status.json" + result = subprocess.run( + [sys.executable, str(pathlib.Path(__file__).with_name('run_coverage_ut.py')), + '--report', str(report), '--progress', str(root/'progress.txt'), + '--status', str(status), '--timeout-diagnostics', str(root/'timeout.txt'), + '--timeout-seconds', '10', '--', + sys.executable, str(pathlib.Path(__file__).with_name('run_coverage_packages.py')), + '--profile', str(profile), '--coverpkg', 'example/a', '--parallel', '8', + EMBED, 'example/a'], + env={**os.environ, 'PATH': str(root)+os.pathsep+os.environ['PATH']}, + capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stderr, '') + events = [json.loads(line) for line in report.read_text().splitlines()] + self.assertEqual([e['Package'] for e in events], ['example/a']) + self.assertTrue(all(e['Action']=='pass' for e in events)) + self.assertEqual(json.loads(status.read_text())['result'], 'passed') + self.assertEqual(profile.with_suffix('.stderr.log').read_text().count('warning:'), 1) + + +if __name__ == "__main__": + unittest.main()