sdkharness: repository-owned leaf checks refuse to run outside a loopback simulator

.sdkharness/README.md

@@ -14,12 +14,74 @@ SDKHARNESS_RESULT	health	golden-path	PASS	-
 ```
 
 The customer-health, conformance, and resilience executables accept only
-literal IPv4 loopback HTTP simulator URLs. They import `b2sdk` from this exact
-checkout and never use a production B2 endpoint or real credentials.
+literal IPv4 loopback HTTP simulator URLs (`http://127.0.0.1:<port>`; `localhost`,
+`::1` and DNS names are refused). They import `b2sdk` from this exact checkout
+and never use a production B2 endpoint or real credentials: every check uses
+only the fixed simulator credential `test-key-id` / `test-key`, and the health
+check refuses any other `B2_TEST_APPLICATION_KEY*` pair before it reaches the
+SDK. The individual conformance and resilience check files refuse to run at all
+unless they are given a loopback simulator URL, so run them through the
+dispatchers below, never with real `B2_*` values in the environment.
 
 Conformance owns 33 capability checks and resilience owns 16 injected-fault
 checks. Their small dispatchers validate the invocation, run the selected
 repository-owned assertion, and translate its standing verdict into the
 five-field `SDKHARNESS_RESULT` record. The central harness continues to own
 scenario selection, simulator lifecycle, fleet evidence, issue reconciliation,
 reporting, and notification.
+
+## Run one check locally
+
+Nothing here touches B2. You need Python 3.10+ and Node 22+ (for the simulator).
+
+```bash
+# 1. This checkout, in a virtualenv (the checks import b2sdk from here)
+python -m venv .venv && . .venv/bin/activate && pip install -e .
+
+# 2. A local simulator (any one of these; it needs access to backblaze-labs/b2-simulator)
+git clone https://github.com/backblaze-labs/b2-simulator /tmp/b2-simulator
+node /tmp/b2-simulator/bin/simulator/serve.mjs --control > /tmp/sim.out 2>&1 &    # prints the URLs
+SIM_PID=$!
+# ...or use the simulator embedded in the harness: sdkharness/bin/simulator/serve.mjs
+
+# 3. Wait until it has printed all three listener lines (http, https, control), then read them
+for _ in $(seq 100); do
+  [ "$(grep -c '^SIMULATOR-' /tmp/sim.out)" -ge 3 ] && break
+  kill -0 "$SIM_PID" 2>/dev/null || { echo 'simulator exited:'; cat /tmp/sim.out; break; }
+  sleep 0.1
+done
+export SDKHARNESS_SIMULATOR_URL=$(sed -n 's/^SIMULATOR-LISTENING \(http:.*\)/\1/p' /tmp/sim.out)
+export SDKHARNESS_SIMULATOR_HTTPS_URL=$(sed -n 's/^SIMULATOR-LISTENING \(https:.*\)/\1/p' /tmp/sim.out)
+export SDKHARNESS_SIMULATOR_CONTROL_URL=$(sed -n 's/^SIMULATOR-CONTROL \(.*\)/\1/p' /tmp/sim.out)
+export SDKHARNESS_SIMULATOR_CA=/tmp/b2-simulator/bin/simulator/loopback-cert.pem
+```
+
+The standalone simulator also exports the same values as `B2SIM_URL`,
+`B2SIM_HTTPS_URL`, `B2SIM_CONTROL_URL` and `B2SIM_CA` (its `bin/lib/simulator.sh`
+helper); the checks read the `SDKHARNESS_SIMULATOR_*` names above.
+
+```bash
+# conformance (one capability)
+SDKHARNESS_TEST_LEVEL=conformance SDKHARNESS_SCENARIO=files.upload .sdkharness/tests/run-conformance
+
+# resilience (one injected fault; needs the control URL, i.e. serve.mjs --control)
+SDKHARNESS_TEST_LEVEL=resilience SDKHARNESS_SCENARIO=api.backoff_503 .sdkharness/tests/run-resilience
+
+# customer health (needs a bucket in the simulator first)
+python - <<'PY'
+import os
+from b2sdk.v3 import B2Api, InMemoryAccountInfo
+api = B2Api(InMemoryAccountInfo())
+api.authorize_account('test-key-id', 'test-key', realm=os.environ['SDKHARNESS_SIMULATOR_URL'])
+api.create_bucket('sdkharness-healthcheck', 'allPrivate')
+PY
+HEALTHCHECK_REALM_URL=$SDKHARNESS_SIMULATOR_URL B2_TEST_APPLICATION_KEY_ID=test-key-id \
+  B2_TEST_APPLICATION_KEY=test-key B2_BUCKET_NAME=sdkharness-healthcheck \
+  .sdkharness/tests/health-golden-path
+```
+
+When you are done, stop the simulator with `kill "$SIM_PID"`.
+
+Each prints one `SDKHARNESS_RESULT` line. Scenario ids are in `tests.tsv`.
+A `FAIL` for `api.retry_after_503`, `upload.retry_408` is a known SDK finding, not a setup problem.
+

.sdkharness/tests/conformance/bucket.cors

@@ -37,6 +37,9 @@ import os
 import sys
 import uuid
 
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'lib'))
+from loopback_guard import refusal  # noqa: E402
+
 SLUG = 'b2-sdk-python'
 CAPABILITY = 'bucket.cors'
 RULE = {
@@ -63,8 +66,6 @@ REASONS = (
 # own server (bin/simulator/serve.mjs), whose URL the runner exports as
 # CONFORMANCE_SIMULATOR_URL; unset, there is nothing to reach: no-realm-option.
 REALMS = {
-    'staging': 'staging',
-    'production': 'production',
     'simulator': os.environ.get('CONFORMANCE_SIMULATOR_URL') or None,
 }
 
@@ -78,7 +79,7 @@ def credential(realm):
     """The (key id, key) pair for a realm: the fixed one at @simulator."""
     if realm == REALMS['simulator']:
         return SIM_CREDENTIAL
-    return os.environ.get('B2_APPLICATION_KEY_ID'), os.environ.get('B2_APPLICATION_KEY')
+    raise AssertionError('conformance checks use only the fixed simulator credential')
 
 
 class Amber(Exception):
@@ -299,7 +300,11 @@ def run(target: str) -> None:
 
 
 def main() -> int:
-    target = os.environ.get('CONFORMANCE_TARGET', 'staging')
+    target = os.environ.get('CONFORMANCE_TARGET', 'simulator')
+    refused = refusal('conformance')
+    if refused:
+        say(target, f'FAIL ({refused})')
+        return 1
     try:
         run(target)
     except Amber as amber:

.sdkharness/tests/conformance/bucket.crud

@@ -30,6 +30,9 @@ import os
 import sys
 import uuid
 
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'lib'))
+from loopback_guard import refusal  # noqa: E402
+
 SLUG = 'b2-sdk-python'
 CAPABILITY = 'bucket.crud'
 
@@ -48,8 +51,6 @@ REASONS = (
 # own server (bin/simulator/serve.mjs), whose URL the runner exports as
 # CONFORMANCE_SIMULATOR_URL; unset, there is nothing to reach: no-realm-option.
 REALMS = {
-    'staging': 'staging',
-    'production': 'production',
     'simulator': os.environ.get('CONFORMANCE_SIMULATOR_URL') or None,
 }
 
@@ -63,7 +64,7 @@ def credential(realm):
     """The (key id, key) pair for a realm: the fixed one at @simulator."""
     if realm == REALMS['simulator']:
         return SIM_CREDENTIAL
-    return os.environ.get('B2_APPLICATION_KEY_ID'), os.environ.get('B2_APPLICATION_KEY')
+    raise AssertionError('conformance checks use only the fixed simulator credential')
 
 
 class Amber(Exception):
@@ -283,7 +284,11 @@ def run(target: str) -> None:
 
 
 def main() -> int:
-    target = os.environ.get('CONFORMANCE_TARGET', 'staging')
+    target = os.environ.get('CONFORMANCE_TARGET', 'simulator')
+    refused = refusal('conformance')
+    if refused:
+        say(target, f'FAIL ({refused})')
+        return 1
     try:
         run(target)
     except Amber as amber:

.sdkharness/tests/conformance/bucket.lifecycle

@@ -28,6 +28,9 @@ import os
 import sys
 import uuid
 
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'lib'))
+from loopback_guard import refusal  # noqa: E402
+
 SLUG = 'b2-sdk-python'
 CAPABILITY = 'bucket.lifecycle'
 RULE = {'fileNamePrefix': 'st/', 'daysFromHidingToDeleting': 1}
@@ -48,8 +51,6 @@ REASONS = (
 # own server (bin/simulator/serve.mjs), whose URL the runner exports as
 # CONFORMANCE_SIMULATOR_URL; unset, there is nothing to reach: no-realm-option.
 REALMS = {
-    'staging': 'staging',
-    'production': 'production',
     'simulator': os.environ.get('CONFORMANCE_SIMULATOR_URL') or None,
 }
 
@@ -63,7 +64,7 @@ def credential(realm):
     """The (key id, key) pair for a realm: the fixed one at @simulator."""
     if realm == REALMS['simulator']:
         return SIM_CREDENTIAL
-    return os.environ.get('B2_APPLICATION_KEY_ID'), os.environ.get('B2_APPLICATION_KEY')
+    raise AssertionError('conformance checks use only the fixed simulator credential')
 
 
 class Amber(Exception):
@@ -271,7 +272,11 @@ def run(target: str) -> None:
 
 
 def main() -> int:
-    target = os.environ.get('CONFORMANCE_TARGET', 'staging')
+    target = os.environ.get('CONFORMANCE_TARGET', 'simulator')
+    refused = refusal('conformance')
+    if refused:
+        say(target, f'FAIL ({refused})')
+        return 1
     try:
         run(target)
     except Amber as amber:

.sdkharness/tests/conformance/bucket.notification_rules

@@ -28,6 +28,9 @@ import string
 import sys
 import uuid
 
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'lib'))
+from loopback_guard import refusal  # noqa: E402
+
 SLUG = 'b2-sdk-python'
 CAPABILITY = 'bucket.notification_rules'
 RULE_NAME = 'sdkharness-conf'
@@ -49,8 +52,6 @@ REASONS = (
 # own server (bin/simulator/serve.mjs), whose URL the runner exports as
 # CONFORMANCE_SIMULATOR_URL; unset, there is nothing to reach: no-realm-option.
 REALMS = {
-    'staging': 'staging',
-    'production': 'production',
     'simulator': os.environ.get('CONFORMANCE_SIMULATOR_URL') or None,
 }
 
@@ -64,7 +65,7 @@ def credential(realm):
     """The (key id, key) pair for a realm: the fixed one at @simulator."""
     if realm == REALMS['simulator']:
         return SIM_CREDENTIAL
-    return os.environ.get('B2_APPLICATION_KEY_ID'), os.environ.get('B2_APPLICATION_KEY')
+    raise AssertionError('conformance checks use only the fixed simulator credential')
 
 
 class Amber(Exception):
@@ -291,7 +292,11 @@ def run(target: str) -> None:
 
 
 def main() -> int:
-    target = os.environ.get('CONFORMANCE_TARGET', 'staging')
+    target = os.environ.get('CONFORMANCE_TARGET', 'simulator')
+    refused = refusal('conformance')
+    if refused:
+        say(target, f'FAIL ({refused})')
+        return 1
     try:
         run(target)
     except Amber as amber:

.sdkharness/tests/conformance/bucket.replication_config

@@ -28,6 +28,9 @@ import os
 import sys
 import uuid
 
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'lib'))
+from loopback_guard import refusal  # noqa: E402
+
 SLUG = 'b2-sdk-python'
 CAPABILITY = 'bucket.replication_config'
 RULE_NAME = 'sdkharness-conf'
@@ -60,8 +63,6 @@ REASONS = (
 # own server (bin/simulator/serve.mjs), whose URL the runner exports as
 # CONFORMANCE_SIMULATOR_URL; unset, there is nothing to reach: no-realm-option.
 REALMS = {
-    'staging': 'staging',
-    'production': 'production',
     'simulator': os.environ.get('CONFORMANCE_SIMULATOR_URL') or None,
 }
 
@@ -75,7 +76,7 @@ def credential(realm):
     """The (key id, key) pair for a realm: the fixed one at @simulator."""
     if realm == REALMS['simulator']:
         return SIM_CREDENTIAL
-    return os.environ.get('B2_APPLICATION_KEY_ID'), os.environ.get('B2_APPLICATION_KEY')
+    raise AssertionError('conformance checks use only the fixed simulator credential')
 
 
 class Amber(Exception):
@@ -319,7 +320,11 @@ def run(target: str) -> None:
 
 
 def main() -> int:
-    target = os.environ.get('CONFORMANCE_TARGET', 'staging')
+    target = os.environ.get('CONFORMANCE_TARGET', 'simulator')
+    refused = refusal('conformance')
+    if refused:
+        say(target, f'FAIL ({refused})')
+        return 1
     try:
         run(target)
     except Amber as amber:

.sdkharness/tests/conformance/bucket.replication_helper

@@ -33,6 +33,9 @@ import os
 import sys
 import uuid
 
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'lib'))
+from loopback_guard import refusal  # noqa: E402
+
 SLUG = 'b2-sdk-python'
 CAPABILITY = 'bucket.replication_helper'
 RULE_NAME = 'sdkharness-conf'
@@ -55,8 +58,6 @@ REASONS = (
 # own server (bin/simulator/serve.mjs), whose URL the runner exports as
 # CONFORMANCE_SIMULATOR_URL; unset, there is nothing to reach: no-realm-option.
 REALMS = {
-    'staging': 'staging',
-    'production': 'production',
     'simulator': os.environ.get('CONFORMANCE_SIMULATOR_URL') or None,
 }
 
@@ -70,7 +71,7 @@ def credential(realm):
     """The (key id, key) pair for a realm: the fixed one at @simulator."""
     if realm == REALMS['simulator']:
         return SIM_CREDENTIAL
-    return os.environ.get('B2_APPLICATION_KEY_ID'), os.environ.get('B2_APPLICATION_KEY')
+    raise AssertionError('conformance checks use only the fixed simulator credential')
 
 
 class Amber(Exception):
@@ -335,7 +336,11 @@ def run(target: str) -> None:
 
 
 def main() -> int:
-    target = os.environ.get('CONFORMANCE_TARGET', 'staging')
+    target = os.environ.get('CONFORMANCE_TARGET', 'simulator')
+    refused = refusal('conformance')
+    if refused:
+        say(target, f'FAIL ({refused})')
+        return 1
     try:
         run(target)
     except Amber as amber:

.sdkharness/tests/conformance/client.auth_persistence

@@ -34,6 +34,9 @@ import sys
 import tempfile
 import uuid
 
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'lib'))
+from loopback_guard import refusal  # noqa: E402
+
 SLUG = 'b2-sdk-python'
 CAPABILITY = 'client.auth_persistence'
 
@@ -52,8 +55,6 @@ REASONS = (
 # own server (bin/simulator/serve.mjs), whose URL the runner exports as
 # CONFORMANCE_SIMULATOR_URL; unset, there is nothing to reach: no-realm-option.
 REALMS = {
-    'staging': 'staging',
-    'production': 'production',
     'simulator': os.environ.get('CONFORMANCE_SIMULATOR_URL') or None,
 }
 
@@ -67,7 +68,7 @@ def credential(realm):
     """The (key id, key) pair for a realm: the fixed one at @simulator."""
     if realm == REALMS['simulator']:
         return SIM_CREDENTIAL
-    return os.environ.get('B2_APPLICATION_KEY_ID'), os.environ.get('B2_APPLICATION_KEY')
+    raise AssertionError('conformance checks use only the fixed simulator credential')
 
 
 class Amber(Exception):
@@ -334,7 +335,11 @@ def run(target: str) -> None:
 
 
 def main() -> int:
-    target = os.environ.get('CONFORMANCE_TARGET', 'staging')
+    target = os.environ.get('CONFORMANCE_TARGET', 'simulator')
+    refused = refusal('conformance')
+    if refused:
+        say(target, f'FAIL ({refused})')
+        return 1
     try:
         run(target)
     except Amber as amber: