From c5271e241dc564c576bf0aae2ad2f4c52d3ef6c6 Mon Sep 17 00:00:00 2001 From: scarreras Date: Sat, 3 Oct 2026 21:46:54 -0300 Subject: [PATCH] test(sdkharness): refuse a resilience run on a simulator that already served requests The resilience leaves count the simulator's whole request journal, which has no reset. On a simulator shared by several scenarios, upload.cap_exceeded_403 read earlier scenarios' b2_upload_file entries as its own and failed with N calls, and api.retry_after_503 counted earlier injected 503s. The dispatcher now FAILs with a configuration reason when the journal is not empty. Co-Authored-By: Claude Sonnet 5.5 --- .sdkharness/README.md | 5 +- .sdkharness/tests/lib/contract.py | 29 ++++++++++ ...silience-fresh-simulator.infrastructure.md | 1 + ...harness_conformance_resilience_contract.py | 53 +++++++++++++++++++ 4 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 changelog.d/+sdkharness-resilience-fresh-simulator.infrastructure.md diff --git a/.sdkharness/README.md b/.sdkharness/README.md index 0bfe3de1..6a1a55d4 100644 --- a/.sdkharness/README.md +++ b/.sdkharness/README.md @@ -64,7 +64,10 @@ helper); the checks read the `SDKHARNESS_SIMULATOR_*` names above. # conformance (one capability) SDKHARNESS_TEST_LEVEL=conformance SDKHARNESS_SCENARIO=files.upload .sdkharness/tests/run-conformance -# resilience (one injected fault; needs the control URL, i.e. serve.mjs --control) +# resilience (one injected fault; needs the control URL, i.e. serve.mjs --control). +# Start a FRESH simulator for every scenario: the simulator keeps one request journal +# with no reset, the leaves count it, and the dispatcher FAILs a simulator that already +# served requests. SDKHARNESS_TEST_LEVEL=resilience SDKHARNESS_SCENARIO=api.backoff_503 .sdkharness/tests/run-resilience # customer health (needs a bucket in the simulator first) diff --git a/.sdkharness/tests/lib/contract.py b/.sdkharness/tests/lib/contract.py index 86da62df..3841c641 100644 --- a/.sdkharness/tests/lib/contract.py +++ b/.sdkharness/tests/lib/contract.py @@ -2,10 +2,12 @@ from __future__ import annotations +import json import os import re import subprocess import sys +import urllib.request from collections.abc import Mapping from pathlib import Path from urllib.parse import urlsplit @@ -100,6 +102,31 @@ def translate(level: str, scenario: str, returncode: int, output: str) -> tuple[ return 'FAIL', 'contract: malformed standing verdict', 1 +def require_fresh_simulator(control_url: str) -> None: + """Refuse a simulator that already served requests. + + The resilience leaves read the simulator's whole request journal and count + its entries (for example every b2_upload_file), and the simulator has no + journal reset. Run on a simulator an earlier scenario used, a leaf reads + that scenario's requests as its own and reports a bogus SDK verdict + (`upload.cap_exceeded_403`: "N b2_upload_file calls"). Start one control + simulator per scenario, as the harness runner does. + """ + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + try: + with opener.open(f'{control_url.rstrip("/")}/journal', timeout=10) as response: + entries = json.load(response)['entries'] + except Exception as error: + raise ContractFailure( + f'configuration: cannot read the simulator journal ({type(error).__name__})' + ) from error + if entries: + raise ContractFailure( + f'configuration: the simulator already served {len(entries)} request(s); ' + 'resilience scenarios need one fresh --control simulator per scenario' + ) + + def run(level: str, environment: Mapping[str, str] = os.environ) -> int: scenario = environment.get('SDKHARNESS_SCENARIO', 'unknown') try: @@ -107,6 +134,8 @@ def run(level: str, environment: Mapping[str, str] = os.environ) -> int: executable = SCENARIO_ROOT / level / scenario if not executable.is_file() or not os.access(executable, os.X_OK): raise ContractFailure('configuration: scenario executable is unavailable') + if level == 'resilience': + require_fresh_simulator(environment['SDKHARNESS_SIMULATOR_CONTROL_URL']) completed = subprocess.run( [str(executable)], cwd=REPOSITORY_ROOT, diff --git a/changelog.d/+sdkharness-resilience-fresh-simulator.infrastructure.md b/changelog.d/+sdkharness-resilience-fresh-simulator.infrastructure.md new file mode 100644 index 00000000..5fd122d2 --- /dev/null +++ b/changelog.d/+sdkharness-resilience-fresh-simulator.infrastructure.md @@ -0,0 +1 @@ +The repository-owned resilience dispatcher refuses a simulator that has already served requests, instead of letting a leaf count an earlier scenario's journal entries (for example `upload.cap_exceeded_403` reporting extra `b2_upload_file` calls) and report a false SDK failure. diff --git a/test/unit/test_sdkharness_conformance_resilience_contract.py b/test/unit/test_sdkharness_conformance_resilience_contract.py index 15d59f77..d1845ddc 100644 --- a/test/unit/test_sdkharness_conformance_resilience_contract.py +++ b/test/unit/test_sdkharness_conformance_resilience_contract.py @@ -180,3 +180,56 @@ def test_leaf_checks_name_no_external_realm_or_ambient_credentials(): assert "os.environ.get('B2_APPLICATION_KEY" not in text, leaf.name assert "os.environ['B2_APPLICATION_KEY" not in text, leaf.name assert 'refusal(' in text, leaf.name + + +class _JournalServer: + """A loopback server answering GET /journal with a fixed entry list.""" + + def __init__(self, entries): + import json + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + body = json.dumps({'entries': entries}).encode() + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('content-length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + self.server = HTTPServer(('127.0.0.1', 0), Handler) + self.url = f'http://127.0.0.1:{self.server.server_address[1]}' + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def close(self): + self.server.shutdown() + self.server.server_close() + + +def test_resilience_refuses_a_simulator_that_already_served_requests(capsys): + contract = load_contract() + server = _JournalServer([{'seq': 1, 'endpoint': 'b2_upload_file', 'status': 200}]) + try: + env = environment('resilience', 'upload.cap_exceeded_403') + env['SDKHARNESS_SIMULATOR_CONTROL_URL'] = server.url + assert contract.run('resilience', env) == 1 + finally: + server.close() + out = capsys.readouterr().out + assert 'FAIL' in out and 'already served 1 request(s)' in out + + +def test_fresh_simulator_is_accepted_and_unreadable_journal_is_a_failure(): + contract = load_contract() + server = _JournalServer([]) + try: + contract.require_fresh_simulator(server.url) + finally: + server.close() + with pytest.raises(contract.ContractFailure, match='cannot read the simulator journal'): + contract.require_fresh_simulator('http://127.0.0.1:1')