Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/actionlint.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
self-hosted-runner:
labels:
- cs-large-7c14g
- amd64-mo-shanghai-8c16g
- amd64-mo-guangzhou-2xlarge16
- amd64-mo-guangzhou-2xlarge32
Expand Down
18 changes: 18 additions & 0 deletions .github/workflows/check-action-file.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,18 @@ on:
- 'scripts/select_coverage_artifacts.py'
- 'scripts/test_select_coverage_artifacts.py'
- 'scripts/test_coverage_native_runtime.py'
- 'scripts/coverage_skip_manifest.py'
- 'scripts/test_coverage_skip_manifest.py'
- 'scripts/test_cs_s3_coverage.py'
- 'scripts/coverage_two_groups.py'
- 'scripts/coverage_two_groups_merge.py'
- 'scripts/coverage_group_collect.py'
- 'scripts/run_coverage_packages.py'
- 'scripts/coverage_timing.py'
- 'scripts/test_coverage_two_groups.py'
- 'scripts/test_coverage_two_groups_merge.py'
- 'scripts/test_coverage_group_collect.py'
- 'scripts/test_run_coverage_packages.py'
- 'scripts/test_merge_trigger_tke_subject.py'
- 'scripts/compact_coverage_profile.go'
- 'scripts/compact_coverage_profile_test.go'
Expand Down Expand Up @@ -103,6 +115,12 @@ jobs:
run: python3 scripts/test_select_coverage_artifacts.py -v
- name: Test coverage native runtime paths
run: python3 scripts/test_coverage_native_runtime.py -v
- name: Test CS S3 routing and skip comparison contracts
timeout-minutes: 3
run: python3 -m unittest scripts.test_cs_s3_coverage scripts.test_coverage_skip_manifest scripts.test_summarize_coverage_ut scripts.test_run_coverage_ut -v
- name: Test three-group scheduling and artifact selection
timeout-minutes: 3
run: PYTHONPATH=scripts python3 -m unittest scripts.test_coverage_two_groups scripts.test_coverage_two_groups_merge scripts.test_coverage_group_collect scripts.test_run_coverage_packages -v
- name: Test TKE merge subject identity contract
run: python3 scripts/test_merge_trigger_tke_subject.py -v
- name: Test SCA license scope selection
Expand Down
297 changes: 221 additions & 76 deletions .github/workflows/coverage-ut.yaml

Large diffs are not rendered by default.

45 changes: 45 additions & 0 deletions scripts/coverage_group_collect.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Select matching latest group evidence and profiles for one Actions run."""
import argparse
import re
from pathlib import Path


def collect(raw_groups, raw_profiles, output, run_id):
if not str(run_id).isdigit():
raise ValueError('invalid run ID')
root = Path(output)
for group in range(3):
def candidates(directory, prefix):
pattern = re.compile(rf'^{prefix}-{group}-(?:generation-)?{run_id}-([1-9][0-9]*)$')
return {int(match.group(1)): p for p in Path(directory).iterdir()
if p.is_dir() and (match := pattern.fullmatch(p.name))}
evidence = candidates(raw_groups, 'ut-coverage-skips-group')
profiles = candidates(raw_profiles, 'ut-coverage-group')
if not evidence or not profiles or max(evidence) != max(profiles):
raise ValueError(f'group {group}: missing or mismatched latest artifacts')
attempt = max(evidence)
group_out = root / 'groups' / str(group)
profile_out = root / 'profiles' / str(group)
group_out.mkdir(parents=True)
profile_out.mkdir(parents=True)
for filename in ('coverage-group-plan.json', 'ut-coverage-skip-manifest.json'):
source = evidence[attempt] / filename
if not source.is_file() or source.is_symlink():
raise ValueError(f'group {group}: missing evidence')
(group_out / filename).symlink_to(source.resolve())
profile = profiles[attempt] / 'ut-coverage.out'
if not profile.is_file() or profile.is_symlink():
raise ValueError(f'group {group}: missing profile')
(profile_out / 'ut-coverage.out').symlink_to(profile.resolve())
print(f'group {group}: selected attempt {attempt}')


if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--groups', required=True)
parser.add_argument('--profiles', required=True)
parser.add_argument('--output', required=True)
parser.add_argument('--run-id', required=True)
args = parser.parse_args()
collect(args.groups, args.profiles, args.output, args.run_id)
121 changes: 121 additions & 0 deletions scripts/coverage_skip_manifest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
#!/usr/bin/env python3
"""Keep small, credential-redacted Coverage skip evidence on success and failure."""

import argparse
from collections import Counter, deque
import json
from pathlib import Path
import re


def redact(text):
text = re.sub(r"https?://\S+", "<endpoint>", text)
text = re.sub(r"\S+@tcp\([^)]*\)\S*", "<database-connection>", text)
text = re.sub(r"(?i)((?:password|secret|token|access[_-]?key|api[_-]?key|key-id|key-secret)\s*[:=]\s*)[^\s,;]+", r"\1<redacted>", text)
return text[:800]


def collect(report, status, source_sha, runner, mode):
counts = Counter()
started_packages = set()
notes = {}
skipped, packages, failed, timeouts = [], [], [], []
malformed = 0
with report.open(errors="replace") as stream:
for line in stream:
try:
event = json.loads(line)
except ValueError:
malformed += 1
continue
if not isinstance(event, dict) or not isinstance(event.get("Action"), str):
malformed += 1
continue
action = event["Action"]
counts[action] += 1
if action == "start":
started_packages.add(event.get("Package", ""))
key = (event.get("Package", ""), event.get("Test", ""))
item = {"package": key[0], "test": key[1]}
if action == "output":
message = event.get("Output", "").strip()
# Keep only Go test messages, not arbitrary application logs.
if re.match(r"[^\s]+\.go:\d+:", message):
notes.setdefault(key, deque(maxlen=8)).append(redact(message))
if re.search(r"panic: test timed out after|^test timed out after", message):
timeouts.append(dict(item, reason=redact(message)))
elif action == "skip":
item["reasons"] = list(notes.pop(key, [])) or [
"skip without message" if key[1] else "package has no test files"
]
(skipped if key[1] else packages).append(item)
elif action in ("pass", "fail"):
notes.pop(key, None)
if action == "fail":
failed.append(item)
return {
"schema": 1, "source_sha": source_sha, "runner": runner, "storage_mode": mode,
"producer_status": status.get("result", "unknown"),
"producer_reason": status.get("reason", "unknown"),
"elapsed_seconds": status.get("elapsed_seconds"),
"malformed_events": malformed, "event_counts": dict(counts),
"started_packages": sorted(started_packages),
"skipped_tests": skipped, "packages_without_tests": packages,
"failed": failed, "package_timeouts": timeouts,
"note": "Real COS/OSS integration is intentionally outside CS UT scope. Absent dynamic storage cases do not appear as skip events.",
}


def compare(baseline, candidate):
def identities(items):
return {(x["package"], x["test"]) for x in items}

old = identities(baseline["skipped_tests"])
new = identities(candidate["skipped_tests"])
added = sorted(new - old)
reasons_changed = []
old_reasons = {(x["package"], x["test"]): x["reasons"] for x in baseline["skipped_tests"]}
for item in candidate["skipped_tests"]:
key = (item["package"], item["test"])
if key in old_reasons and old_reasons[key] != item["reasons"]:
reasons_changed.append(item)
comparable = baseline["source_sha"] == candidate["source_sha"] and bool(baseline["source_sha"])
same_packages = bool(baseline["started_packages"]) and baseline["started_packages"] == candidate["started_packages"]
healthy = all(x["producer_status"] == "passed" and not x["malformed_events"]
and not x["failed"] and not x["package_timeouts"] for x in (baseline, candidate))
return {
"same_source": comparable, "same_package_scope": same_packages, "both_producers_passed": healthy,
"new_skips": [{"package": p, "test": t} for p, t in added],
"removed_skips": [{"package": p, "test": t} for p, t in sorted(old - new)],
"changed_skip_reasons": reasons_changed,
"baseline_skip_tests": len(old), "candidate_skip_tests": len(new),
"accepted": comparable and same_packages and healthy and not added,
"scope": "Existing skipped identities are accepted; real COS/OSS coverage is not required. Changed reasons are retained for review, not treated as new skips.",
}


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--report", type=Path)
parser.add_argument("--status", type=Path)
parser.add_argument("--source-sha")
parser.add_argument("--runner", default="unknown")
parser.add_argument("--mode", default="unknown")
parser.add_argument("--baseline", type=Path)
parser.add_argument("--candidate", type=Path)
parser.add_argument("--output", required=True, type=Path)
args = parser.parse_args()
if args.baseline and args.candidate:
result = compare(json.loads(args.baseline.read_text()), json.loads(args.candidate.read_text()))
exit_code = 0 if result["accepted"] else 1
elif args.report and args.status and args.source_sha:
result = collect(args.report, json.loads(args.status.read_text()), args.source_sha, args.runner, args.mode)
exit_code = 0
else:
parser.error("provide report/status/source-sha or baseline/candidate")
args.output.write_text(json.dumps(result, indent=2) + "\n")
return exit_code


if __name__ == "__main__":
raise SystemExit(main())
54 changes: 54 additions & 0 deletions scripts/coverage_timing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/usr/bin/env python3
"""Extract per-package and embedded-cluster admission timings from Go JSON."""
import argparse
import json
import re
from collections import defaultdict
from pathlib import Path

UNIT = {'ns': 1e-9, 'µs': 1e-6, 'us': 1e-6, 'ms': 1e-3, 's': 1, 'm': 60, 'h': 3600}
PART = re.compile(r'(\d+(?:\.\d+)?)(ns|µs|us|ms|s|m|h)')


def seconds(value):
parts = list(PART.finditer(value))
if not parts or ''.join(m.group(0) for m in parts) != value:
raise ValueError('unrecognized Go duration: ' + value)
return sum(float(m.group(1)) * UNIT[m.group(2)] for m in parts)


def summarize(path):
packages, waits = {}, defaultdict(list)
parse_errors = 0
with path.open(encoding='utf-8') as source:
for line in source:
try:
event = json.loads(line)
except (ValueError, UnicodeError):
parse_errors += 1
continue
package = event.get('Package')
if not package:
continue
if event.get('Action') in ('pass', 'fail', 'skip') and not event.get('Test'):
packages[package] = {'action': event['Action'], 'elapsed_seconds': event.get('Elapsed')}
output = event.get('Output', '')
if 'MO_UT_SETUP' in output and 'phase=admission-acquire' in output:
found = re.search(r'(?:^|\s)duration=([^\s]+)', output)
if found:
try:
waits[package].append({'test': event.get('Test'), 'seconds': seconds(found.group(1))})
except ValueError:
parse_errors += 1
return {'packages': packages, 'admission_observations': dict(waits), 'parse_errors': parse_errors,
'limitations': 'Admission values cover only emitted embedded-cluster setup records; package times overlap.'}


if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('report', type=Path)
parser.add_argument('output', type=Path)
args = parser.parse_args()
with args.output.open('x', encoding='utf-8') as out:
json.dump(summarize(args.report), out, ensure_ascii=False, indent=2)
out.write('\n')
68 changes: 68 additions & 0 deletions scripts/coverage_two_groups.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""Deterministically partition a complete Go package list into three groups.

Weights are approximate elapsed seconds from one MNL2 Coverage run; they are
only scheduling hints, never an allowlist. Unknown packages are still tested.
"""
import argparse
import hashlib
import json
from pathlib import Path

ROOT = 'github.com/matrixorigin/matrixone/'
WEIGHTS = {
'pkg/tests/sqlintegration': 879, 'pkg/tests/service': 638,
'pkg/tests/issues/isolated': 633, 'pkg/embed': 511,
'pkg/tests/issues': 474, 'pkg/sql/plan': 219,
'pkg/tests/sqlintegration/multicn': 208, 'pkg/vm/engine/test': 189,
'pkg/tests/arrowload': 142, 'pkg/logservice': 127,
'pkg/tests/upgrade': 117, 'pkg/tests/partition': 101,
'pkg/tests/dml': 94, 'pkg/vm/engine/tae/db/test': 83,
'optools': 82, 'pkg/shardservice': 57,
'pkg/vm/engine/disttae': 54, 'pkg/tests/shard': 54,
'pkg/lockservice': 50, 'pkg/frontend': 46,
}
FILE_SERVICE = ROOT + 'pkg/fileservice'
EMBED = ROOT + 'pkg/embed'


def plan(packages):
if not packages or len(set(packages)) != len(packages):
raise ValueError('empty or duplicate package scope')
if FILE_SERVICE not in packages or EMBED not in packages:
raise ValueError('required S3 or embed test package is missing')
# Seed heavy packages into different groups; shared-pool jobs can land in
# any environment, so no group is assumed to use a particular runner.
groups = [[FILE_SERVICE], [], [EMBED]]
weights = [5, 0, WEIGHTS['pkg/embed']]
for package in sorted(set(packages) - {FILE_SERVICE, EMBED},
key=lambda p: (-WEIGHTS.get(p.removeprefix(ROOT), 5), p)):
index = min(range(3), key=lambda i: (weights[i], i))
groups[index].append(package)
weights[index] += WEIGHTS.get(package.removeprefix(ROOT), 5)
groups = [sorted(group) for group in groups]
if not all(groups) or sum(len(set(g)) for g in groups) != len(set().union(*map(set, groups))) or set().union(*map(set, groups)) != set(packages):
raise ValueError('invalid partition')
scope = sorted(packages)
digest = hashlib.sha256(('\n'.join(scope) + '\n').encode()).hexdigest()
return {'schema': 1, 'scope_sha256': digest, 'scope_count': len(scope),
'groups': groups, 'estimated_weight': weights}


def main():
parser = argparse.ArgumentParser()
parser.add_argument('--packages', required=True, type=Path)
parser.add_argument('--group', required=True, type=int, choices=(0, 1, 2))
parser.add_argument('--manifest', required=True, type=Path)
parser.add_argument('--selected', required=True, type=Path)
args = parser.parse_args()
result = plan(args.packages.read_text().splitlines())
result['selected_group'] = args.group
args.manifest.write_text(json.dumps(result, sort_keys=True) + '\n')
args.selected.write_text('\n'.join(result['groups'][args.group]) + '\n')
print(f"group={args.group} selected={len(result['groups'][args.group])} "
f"scope={result['scope_count']} scope_sha256={result['scope_sha256']}")


if __name__ == '__main__':
main()
48 changes: 48 additions & 0 deletions scripts/coverage_two_groups_merge.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Validate complete group evidence before merging coverage profiles."""
import argparse
import json
from pathlib import Path


def validate(root, expected_sha=None):
plans = [json.loads((root / str(i) / 'coverage-group-plan.json').read_text()) for i in range(3)]
results = [json.loads((root / str(i) / 'ut-coverage-skip-manifest.json').read_text()) for i in range(3)]
for i in range(3):
p, r = plans[i], results[i]
if p['selected_group'] != i or r['producer_status'] != 'passed':
raise ValueError('group identity or producer status mismatch')
if r['malformed_events'] or r['failed'] or r['package_timeouts']:
raise ValueError('group reported failed tests, malformed events, or timeout')
if set(r['started_packages']) != set(p['groups'][i]):
raise ValueError('actual package scope does not match group plan')
if any(plans[0][key] != plan[key] for plan in plans[1:] for key in ('scope_sha256', 'scope_count', 'groups')):
raise ValueError('group plans differ')
groups = list(map(set, plans[0]['groups']))
if len(groups) != 3 or not all(groups) or sum(map(len, groups)) != len(set().union(*groups)) or len(set().union(*groups)) != plans[0]['scope_count']:
raise ValueError('incomplete or overlapping group scope')
if any(results[0]['source_sha'] != result['source_sha'] for result in results[1:]):
raise ValueError('source mismatch')
if expected_sha and results[0]['source_sha'] != expected_sha:
raise ValueError('source does not match the PR head')
return results


def main():
parser = argparse.ArgumentParser()
parser.add_argument('root', type=Path)
parser.add_argument('output', type=Path)
parser.add_argument('--expected-sha', required=True)
args = parser.parse_args()
results = validate(args.root, args.expected_sha)
result = dict(results[0])
for key in ('started_packages', 'skipped_tests', 'packages_without_tests', 'failed', 'package_timeouts'):
result[key] = sum((item[key] for item in results), [])
result['started_packages'] = sorted(result['started_packages'])
result['runner'] = 'three isolated runner jobs'
args.output.write_text(json.dumps(result, indent=2) + '\n')
print(f"Validated {len(result['started_packages'])} packages across all groups")


if __name__ == '__main__':
main()
Loading
Loading