test(sdkharness): refuse a resilience run on a simulator that already served requests

.sdkharness/README.md

@@ -64,7 +64,10 @@ helper); the checks read the `SDKHARNESS_SIMULATOR_*` names above.
 # conformance (one capability)
 SDKHARNESS_TEST_LEVEL=conformance SDKHARNESS_SCENARIO=files.upload .sdkharness/tests/run-conformance
 
-# resilience (one injected fault; needs the control URL, i.e. serve.mjs --control)
+# resilience (one injected fault; needs the control URL, i.e. serve.mjs --control).
+# Start a FRESH simulator for every scenario: the simulator keeps one request journal
+# with no reset, the leaves count it, and the dispatcher FAILs a simulator that already
+# served requests.
 SDKHARNESS_TEST_LEVEL=resilience SDKHARNESS_SCENARIO=api.backoff_503 .sdkharness/tests/run-resilience
 
 # customer health (needs a bucket in the simulator first)

.sdkharness/tests/lib/contract.py

@@ -2,10 +2,12 @@
 
 from __future__ import annotations
 
+import json
 import os
 import re
 import subprocess
 import sys
+import urllib.request
 from collections.abc import Mapping
 from pathlib import Path
 from urllib.parse import urlsplit
@@ -100,13 +102,40 @@ def translate(level: str, scenario: str, returncode: int, output: str) -> tuple[
     return 'FAIL', 'contract: malformed standing verdict', 1
 
 
+def require_fresh_simulator(control_url: str) -> None:
+    """Refuse a simulator that already served requests.
+
+    The resilience leaves read the simulator's whole request journal and count
+    its entries (for example every b2_upload_file), and the simulator has no
+    journal reset. Run on a simulator an earlier scenario used, a leaf reads
+    that scenario's requests as its own and reports a bogus SDK verdict
+    (`upload.cap_exceeded_403`: "N b2_upload_file calls"). Start one control
+    simulator per scenario, as the harness runner does.
+    """
+    opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+    try:
+        with opener.open(f'{control_url.rstrip("/")}/journal', timeout=10) as response:
+            entries = json.load(response)['entries']
+    except Exception as error:
+        raise ContractFailure(
+            f'configuration: cannot read the simulator journal ({type(error).__name__})'
+        ) from error
+    if entries:
+        raise ContractFailure(
+            f'configuration: the simulator already served {len(entries)} request(s); '
+            'resilience scenarios need one fresh --control simulator per scenario'
+        )
+
+
 def run(level: str, environment: Mapping[str, str] = os.environ) -> int:
     scenario = environment.get('SDKHARNESS_SCENARIO', 'unknown')
     try:
         scenario, _ = validate_environment(level, environment)
         executable = SCENARIO_ROOT / level / scenario
         if not executable.is_file() or not os.access(executable, os.X_OK):
             raise ContractFailure('configuration: scenario executable is unavailable')
+        if level == 'resilience':
+            require_fresh_simulator(environment['SDKHARNESS_SIMULATOR_CONTROL_URL'])
         completed = subprocess.run(
             [str(executable)],
             cwd=REPOSITORY_ROOT,

changelog.d/+sdkharness-resilience-fresh-simulator.infrastructure.md

@@ -0,0 +1 @@
+The repository-owned resilience dispatcher refuses a simulator that has already served requests, instead of letting a leaf count an earlier scenario's journal entries (for example `upload.cap_exceeded_403` reporting extra `b2_upload_file` calls) and report a false SDK failure.

test/unit/test_sdkharness_conformance_resilience_contract.py

@@ -180,3 +180,56 @@ def test_leaf_checks_name_no_external_realm_or_ambient_credentials():
         assert "os.environ.get('B2_APPLICATION_KEY" not in text, leaf.name
         assert "os.environ['B2_APPLICATION_KEY" not in text, leaf.name
         assert 'refusal(' in text, leaf.name
+
+
+class _JournalServer:
+    """A loopback server answering GET /journal with a fixed entry list."""
+
+    def __init__(self, entries):
+        import json
+        import threading
+        from http.server import BaseHTTPRequestHandler, HTTPServer
+
+        body = json.dumps({'entries': entries}).encode()
+
+        class Handler(BaseHTTPRequestHandler):
+            def do_GET(self):
+                self.send_response(200)
+                self.send_header('content-length', str(len(body)))
+                self.end_headers()
+                self.wfile.write(body)
+
+            def log_message(self, *args):
+                pass
+
+        self.server = HTTPServer(('127.0.0.1', 0), Handler)
+        self.url = f'http://127.0.0.1:{self.server.server_address[1]}'
+        threading.Thread(target=self.server.serve_forever, daemon=True).start()
+
+    def close(self):
+        self.server.shutdown()
+        self.server.server_close()
+
+
+def test_resilience_refuses_a_simulator_that_already_served_requests(capsys):
+    contract = load_contract()
+    server = _JournalServer([{'seq': 1, 'endpoint': 'b2_upload_file', 'status': 200}])
+    try:
+        env = environment('resilience', 'upload.cap_exceeded_403')
+        env['SDKHARNESS_SIMULATOR_CONTROL_URL'] = server.url
+        assert contract.run('resilience', env) == 1
+    finally:
+        server.close()
+    out = capsys.readouterr().out
+    assert 'FAIL' in out and 'already served 1 request(s)' in out
+
+
+def test_fresh_simulator_is_accepted_and_unreadable_journal_is_a_failure():
+    contract = load_contract()
+    server = _JournalServer([])
+    try:
+        contract.require_fresh_simulator(server.url)
+    finally:
+        server.close()
+    with pytest.raises(contract.ContractFailure, match='cannot read the simulator journal'):
+        contract.require_fresh_simulator('http://127.0.0.1:1')