diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 90d8b9e15d717..6382142ca71ad 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -3345,7 +3345,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas s3 storage, parallel)' --workflow "MasterCI" --ci --timestamp stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }} name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)" @@ -3396,7 +3396,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)' --workflow "MasterCI" --ci --timestamp stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }} name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7924f9837029b..1a9800ccf9663 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -2347,7 +2347,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas s3 storage, parallel)' --workflow "PR" --ci --timestamp stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }} name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)" @@ -2396,7 +2396,7 @@ jobs: PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)' --workflow "PR" --ci --timestamp stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }} name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)" diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index bf1a63d7d667f..807b53cafb205 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -2537,7 +2537,7 @@ jobs: retention-days: 14 stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }} name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)" @@ -2601,7 +2601,7 @@ jobs: retention-days: 14 stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, config_workflow, fast_test] if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }} name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)" diff --git a/ci/defs/altinity_jobs.py b/ci/defs/altinity_jobs.py index 5c8f832bdb038..19a005a9f9bea 100644 --- a/ci/defs/altinity_jobs.py +++ b/ci/defs/altinity_jobs.py @@ -85,7 +85,15 @@ class AltinityJobConfigs: *[ Job.ParamSet( parameter=f"amd_asan_ubsan, cas s3 storage, parallel, {batch}/{total_batches}", - runs_on=RunnerLabels.AMD_MEDIUM_CPU, + # Under ASan the server RSS is mostly per-thread runtime state (fake + # stacks, thread stacks, allocator regions), not query memory, and it + # grows with the number of pool threads for the life of the process. + # On the 32 GiB runner a shard crosses the 0.8 memory-pressure + # threshold of clickhouse-test after ~2 h, and shedding workers does + # not lower RSS, so the shard drains to one worker and hits the job + # timeout. Same remedy as the plain ASan parallel lane: the LARGE + # runner has the same 16 vCPU and twice the RAM. + runs_on=RunnerLabels.AMD_LARGE, requires=[ArtifactNames.CH_AMD_ASAN_UBSAN_GH], ) for total_batches in (2,) diff --git a/docs/en/antalya/cas/architecture/backend.md b/docs/en/antalya/cas/architecture/backend.md index cc0740ea73da7..789661f6999ae 100644 --- a/docs/en/antalya/cas/architecture/backend.md +++ b/docs/en/antalya/cas/architecture/backend.md @@ -34,7 +34,7 @@ different semantics: `PartWriteTxn::ensureBlobPresent` owns `HEAD`, freshness me `publishBlob` only moves the selected bytes. Writer readiness is represented by `BlobDependencyProof`, not by token presence. `Materialized` -means the writer observed a present non-condemned body or completed publication and metadata +means the writer observed a present body with a `Clean` marker or completed publication and metadata reconciliation. `TrustedManifest` means a durable source manifest proves the blob and requires no blob I/O. Pending state and writer tokens are not stored in the dependency record. @@ -125,7 +125,8 @@ battery, skipping the battery used to skip it too, which is exactly why the thir writable generation-token mount; every other backend still skips only its permitted access-check I/O. This is what makes the exact-token delete check below unskippable on an *ordinary* writable mount. Decommissioning a pool member is the deliberate exception — it opens writable with the - battery skipped, because the fail-closed tradeoff inverts there: refusing an ordinary mount costs + battery skipped and logs one line naming what went unverified (versioning, conditional writes, + list-after-write), because the fail-closed tradeoff inverts there: refusing an ordinary mount costs availability and protects data, whereas refusing a decommission strands a pool with a dead replica in it and leaves the operator no way forward. - `checkConditionalWriteSingleAttemptSupport` — inside the battery. Refuses to mount writable unless the underlying @@ -134,6 +135,14 @@ battery, skipping the battery used to skip it too, which is exactly why the thir committed, so retries on the conditional path must be explicit CAS state-machine transitions, not transparent client behavior. +Blob publication is unconditional and does not use that profile, but its object-creating request +(`PutObject`, `CopyObject`, `CompleteMultipartUpload`) is also sent once. On a store with +content-derived ETags, a resent body that lands after GC deleted its incarnation brings that ETag +back, and the pending exact-ETag delete then removes a live body. The request runs on a clone of the +disk client with the disk's `request_timeout_ms`, no adaptive first-attempt timeouts, and the connect +timeout capped as for the control plane. `CreateMultipartUpload`, `UploadPart` and `UploadPartCopy` +keep the disk client's retries. + A third staging check requires `supportsCopyMode(ObjectStorageCopyMode::NativeOnly)` when `cas_staging_backend = s3`. Writable mount fails closed when native same-store copy is unavailable; it does not silently fall back to local staging. Ordinary non-CAS `copyObject` fallback behavior is diff --git a/docs/en/antalya/cas/architecture/blob-protocol.md b/docs/en/antalya/cas/architecture/blob-protocol.md index 54f6399d073a7..1bea888fdeb84 100644 --- a/docs/en/antalya/cas/architecture/blob-protocol.md +++ b/docs/en/antalya/cas/architecture/blob-protocol.md @@ -36,11 +36,11 @@ sequenceDiagram alt body present S3-->>Writer: present, size, backend token t1 Writer->>S3: GET .meta (point read, body never streamed) - alt meta Clean or absent + alt meta Clean Writer->>Writer: record token-free BlobDependencyProof::Materialized (never adopt t1) - else meta Condemned + else meta Condemned or absent Writer->>S3: unconditional publish with fresh envelope - Writer->>S3: reconcile .meta to Clean + Writer->>S3: create or reconcile .meta to Clean end else body absent S3-->>Writer: absent @@ -56,18 +56,29 @@ Ordered steps in `PartWriteTxn::ensureBlobPresent`: still live. 2. **Mandatory observation.** Every physical materialization begins with one blob `HEAD`, regardless of size, provider, staging backend, or whether another writer probably uploaded the same hash. -3. On a hit, the writer reads `.meta`. `Clean` or absent metadata permits adoption; the writer - records a `Materialized` dependency proof without retaining the observed token. `Condemned` - requires a new publication. +3. On a hit, the writer reads `.meta`. Only `Clean` permits adoption; the writer records a + `Materialized` dependency proof without retaining the observed token. `Condemned` or absent + metadata requires a new publication. `GC` deletes `.meta` right after the body, so an absent + marker can follow a delete that landed between the `HEAD` and the read. 4. On a miss, the writer does not read `.meta` before publication. It publishes its own payload unconditionally, then creates or reconciles `.meta` to `Clean`. 5. Streaming publication mints a fresh `incarnation_tag` and can use ordinary multipart. The first publication of an S3-staged source may use native same-store copy only after a miss; a condemned or subsequent publication retags and streams the staged payload. + The request that creates the object (`PutObject`, `CopyObject`, `CompleteMultipartUpload`) is + never resent by the S3 client after an ambiguous outcome; resends after a definite refusal, such as + expired credentials or a redirect, stay. On a store with content-derived ETags, a resent + envelope that lands after GC deleted it would restore the condemned incarnation. Part uploads keep + the client's retries. 6. `BlobSource::beginPublication` consumes the shared, monotonic `publication_attempted` state before backend I/O. A lost response cannot re-enable verbatim copy on a later attempt. -7. Retryable or ambiguous failures restart at `HEAD`. No dependency proof is recorded until a - present non-condemned body was observed, or publication and metadata reconciliation completed. +7. Retryable or ambiguous failures restart at `HEAD`: up to 8 attempts, each with a fresh envelope. + The engine starts no new attempt after the 90-second deadline, but an attempt already started can + run longer: each creating request waits up to the disk's `request_timeout_ms`, and multipart part + uploads keep their own retries. A write can therefore take longer than 90 seconds before it + fails with a retryable error. No dependency + proof is recorded until a present body with a `Clean` marker was observed, or publication and + metadata reconciliation completed. **Two writers uploading identical content** may both observe absence and both publish. The last physical incarnation wins, but the key proves that both payloads have the same logical identity and @@ -151,7 +162,7 @@ sequenceDiagram alt writer republished S3-->>GC: TokenMismatch -- nothing deleted, blob is live at t2 else genuinely dead - S3-->>GC: Deleted -- then drop the .meta only while it is still Condemned at round n or older + S3-->>GC: Deleted -- then drop the .meta only while it is still Condemned at round n+1 or older end ``` @@ -163,7 +174,7 @@ and a writer never trusts a body it did not itself just write. Why this closes the race in both directions: -- A writer that **adopts** a present body must have read a non-`Condemned` marker, and its precommit +- A writer that **adopts** a present body must have read a `Clean` marker, and its precommit edge was durable *before* that read. The next fold therefore sees in-degree ≥ 1 and spares the blob. - A writer that **replaces a condemned incarnation** changes the token. A stale `deleteExact(t1)` then returns @@ -174,9 +185,11 @@ Why this closes the race in both directions: at the entry's round or newer. A `Clean`, absent, older-round or unreadable marker never graduates the entry: `GC` does not throw, it carries the entry and rewrites the marker at the current round. -- The delete job removes the `.meta` only while it is still `Condemned` at the entry's round or - older, and conditionally on the version it read. A marker rewritten `Clean` by a writer, or - condemned again at a later round, stays. +- The delete job removes the `.meta` only while it is still `Condemned` at a round before the + deleting round, and conditionally on the version it read. It also removes a marker the graduation + gate re-stamped. A marker rewritten `Clean` by a writer, or condemned in the deleting round, + stays. The delete job runs best effort: if it fails, the `Condemned` marker outlives its body + until a writer of that content replaces it, which costs storage, not safety. Both directions degrade to a spurious re-upload or a no-op delete. Neither can lose data or leave a dangling manifest entry. @@ -189,8 +202,9 @@ in RAM at a time. Native object storage streams and can use multipart. `.meta` has exactly two states: `Clean` (body present, may be referenced) and `Condemned` (`GC` observed zero in-degree; the body is still present and a writer may replace it). An -*absent* `.meta` reads exactly like `Clean` — there is no third "unaccounted" state in the -stored format; `unaccounted` is a `cas-fsck` classification, not something `GC` ever writes. +*absent* `.meta` never permits adoption: the writer republishes as for an absent body. There is +no third "unaccounted" state in the stored format; `unaccounted` is a `cas-fsck` classification, +not something `GC` ever writes. The record carries `state`, `condemn_round`, and `size`, and deliberately carries **no token**: it is a per-hash hint, not a per-incarnation fact. All safety comes from the body's in-envelope diff --git a/docs/en/antalya/cas/architecture/correctness.md b/docs/en/antalya/cas/architecture/correctness.md index 8260ca02f9b47..6a0ae43069483 100644 --- a/docs/en/antalya/cas/architecture/correctness.md +++ b/docs/en/antalya/cas/architecture/correctness.md @@ -24,7 +24,7 @@ The full model index (source `.tla` files and proof-run records) lives at | Model (`docs/superpowers/models/`) | Invariant it proves | Counterexample it caught | |---|---|---| -| `CaBlobPublishCore.tla` | Split `HEAD` from unconditional publication while preserving logical content, fresh incarnation identity, monotonic `publication_attempted`, fence safety, and readiness only after metadata reconciliation | Eleven sabotage configurations cover condemned adoption, stale exact delete, ambiguous copy/PUT landing, envelope reuse after a later miss, missing precommit/meta reconciliation, fence loss, and wrong-content publication; three witnesses prove the safe paths are reachable | +| `CaBlobPublishCore.tla` | Split `HEAD` from unconditional publication while preserving logical content, fresh incarnation identity, monotonic `publication_attempted`, fence safety, and readiness only after metadata reconciliation | Sabotage configurations cover condemned adoption, stale exact delete, ambiguous copy/PUT landing, envelope reuse after a later miss, missing precommit/meta reconciliation, fence loss, and wrong-content publication; three witnesses prove the safe paths are reachable. The model also lets the S3 client resend an identical envelope below the engine: on a content-ETag store that resend restores a condemned incarnation, and a stale exact delete then removes it, which is why blob creation is never resent | | `CaIncarnationCore.tla` | `INV_NO_DANGLE`, `INV_NO_LOSS`, `INV_NO_RETURN` — the safety spine for the whole GC core | `sab_unconddelete`: replacing the exact-token delete with an unconditional one lets a stale delete kill a replacement incarnation | | `CaBuildRootPrecommit.tla` | `INV_NO_DANGLE_COMMITTED` — a committed manifest never references an absent blob | Reproduces the dangling-manifest hazard exactly: `WriteBlob → AdoptBlob → BuildDie → GcDelete → Commit` with no presence re-check publishes a manifest over a deleted blob | | `CaGcLeaseCore.tla` | `NoFalseSteal` — no leader steals leadership from a live, mid-round incumbent | Without the advisory heartbeat, a frozen `seq` during a round looks identical to a dead leader, and a second leader steals from the alive one | diff --git a/docs/en/antalya/cas/architecture/design-history.md b/docs/en/antalya/cas/architecture/design-history.md index 6b5c371dd86f2..a6d38058b1e74 100644 --- a/docs/en/antalya/cas/architecture/design-history.md +++ b/docs/en/antalya/cas/architecture/design-history.md @@ -21,7 +21,7 @@ the sequence of pivots that produced the architecture described elsewhere in thi | **Integer in-degree refcount** (a mutable counter, incremented per reference, decremented per release) | The decide-to-reference-then-not-yet-durable window let the fold observe in-degree 0 for a still-live blob; a mutable counter also costs a `CAS` round-trip proportional to write volume | A derived count: `GC` folds a multiset of `+`/`-` source-edge deltas, so losing or duplicating a record can only delay reclamation, never accelerate one | | **Extending zero-copy replication instead of a new mechanism** | Zero-copy's structural costs (a commit spanning local disk, S3, and `Keeper`; a mutable refcount) are inherent to its design, not a bug to patch | `CAS` is an alternative to zero-copy, not a replacement: both remain available, `metadata_type = cas` is opt-in per disk, and no existing deployment needs to migrate | | **Per-incarnation body keys** (`blobs/xx/.`, an alternative to the in-body incarnation tag) | A resurrect reusing the condemned incarnation instead of minting a fresh one reintroduced the shared-key race; structurally this was generation-in-the-key again | The in-body `incarnation_tag` plus exact-token body delete, which keeps the generation out of every object key | -| **Meta as the lifecycle linearizer** (a per-hash `.meta` object whose presence/absence *was* the authority for a blob's lifetime) | The marker is a point-read hint only, never consulted by reads; treating it as the linearizer would assert a guarantee the design does not make | The meta stays advisory: the in-body incarnation tag and exact-token delete are the real authority, and an absent meta reads identically to `Clean` | +| **Meta as the lifecycle linearizer** (a per-hash `.meta` object whose presence/absence *was* the authority for a blob's lifetime) | The marker is a point-read hint only, never consulted by reads; treating it as the linearizer would assert a guarantee the design does not make | The meta stays advisory: the in-body incarnation tag and exact-token delete are the real authority, and an absent meta never permits adoption | | **Raw immutable bodies with a three-state tombstone meta** | A resurrect displacing the body forced a terminal-tombstone handshake — a writer↔`GC` liveness coupling that could re-enable data loss | The settled one-key-per-hash design with an in-body incarnation tag | | **Conditional blob creation and conditional staged copy** | Coupled immutable payload publication to provider-specific ETag/generation responses, forced generation-token GCS blobs into a single-`PUT` size cliff, and split the writer into provider/source-specific branches | One provider-neutral state machine: durable precommit, mandatory blob `HEAD`, unconditional publication for absent or condemned bodies, freshness-metadata reconciliation, and explicit `Materialized`/`TrustedManifest` proof | | **A persistent, append-only namespace registry for `GC` discovery** | Never deregistered on drop, so it grew monotonically forever; its fence cost scaled with namespaces ever created, not namespaces live | Discovery from the ref data itself, made safe by two independent coordinates: a durable per-shard incarnation plus a pool-global round | diff --git a/docs/en/antalya/cas/architecture/garbage-collection.md b/docs/en/antalya/cas/architecture/garbage-collection.md index 24c90c6dd670e..a06663cbd5c17 100644 --- a/docs/en/antalya/cas/architecture/garbage-collection.md +++ b/docs/en/antalya/cas/architecture/garbage-collection.md @@ -193,8 +193,9 @@ token of each `mount` object. - **Writes / deletes:** a token-guarded `PUT` per fenced mount (`gc_fenced = true`, `seq + 1`); no deletes - **Safety:** fences only after this leader's *own* monotonic clock has watched the mount's - write-token hold unchanged for `cas_mount_lease_ttl_ms + 5% + cas_mount_renew_period_ms` - (defaults 30 s and 10 s; every server sharing the pool must run the same values); the write is + write-token hold unchanged for `L + 5% + cas_mount_renew_period_ms`, where `L` is the longer of this server's + `cas_mount_lease_ttl_ms` and the lease the slot's body declares (defaults 30 s and 10 s), and only at the + start of a round, so a killed member becomes `fenced` up to one GC round later than that; the write is guarded by that exact token. A completed fence-out stays valid even if this GC later loses leadership. `expires_at_ms` (another host's wall clock) is never trusted. - **Fails the round if:** nothing — a per-mount `PUT` conflict is re-observed and re-classified @@ -691,7 +692,7 @@ flowchart LR D -->|"still zero, .meta absent, Clean, older or unreadable"| C2["carried, .meta rewritten at this round, never throw"] D -->|"current token not equal to t"| SUP["SUPERSEDED -- a writer resurrected, re-condemn the CURRENT token"] G --> E["round n+2, pre-CAS: exact-token DELETE of blob at t"] - E -->|"Deleted or Absent"| F["then drop the .meta if it is still Condemned at round n or older"] + E -->|"Deleted or Absent"| F["then drop the .meta if it is still Condemned from a round before this one"] E -->|TokenMismatch| H["nothing deleted -- live at a newer token, leave the .meta alone"] ``` @@ -703,9 +704,11 @@ round that issues it and replaces an absent marker, `Clean`, or an older-round ` Graduation reads the marker in the round that graduates and accepts only `Condemned` at the entry's round or newer: an older round may belong to another incarnation of the same content. The marker delete after a redelete -removes only `Condemned` at the entry's round or older, conditional on the version it read, so a delayed delete -never removes the marker of a later incarnation. It can leave an orphan `Condemned` marker over an absent body; -the next writer of that hash republishes and clears it, and `cas-fsck` counts it in `meta_without_body`. +removes only `Condemned` from a round before the deleting round, conditional on the version it read. This covers +a marker the graduation gate re-stamped at a round later than the entry's. It can also remove the marker of a +later incarnation; that is safe because a writer adopts only on `Clean` and republishes over an absent marker. +A failed delete job can leave an orphan `Condemned` marker over an absent body; the next writer of that hash +republishes and clears it, and `cas-fsck` counts it in `meta_without_body`. A blob whose in-degree reaches zero in round `n` is retired with `condemn_round = n`; it can graduate to `delete_pending` in round `n+1` at the earliest and be deleted in round `n+2`, so a minimum of two full rounds @@ -764,7 +767,7 @@ logs: | `PUT` run segments | 1 per non-pure-carry shard, plus 1 fold seal | | `HEAD` blobs | 1 per newly condemned | | `GET` blob `.meta` at graduation | 1 per entry the graduation budget admits, plus 1 `GET` and 1 `PUT` per refusal (the rewrite job re-reads the marker before its conditional `PUT`) | -| `GET` + conditional `DELETE` blob `.meta` after a redelete | 1 `GET` per `Deleted` or `Absent` outcome; the `DELETE` only while the marker is `Condemned` at the entry's round or older | +| `GET` + conditional `DELETE` blob `.meta` after a redelete | 1 `GET` per `Deleted` or `Absent` outcome; the `DELETE` only while the marker is `Condemned` from a round before the deleting round | | Blob `HEAD` + conditional `DELETE` | 1 `HEAD` per `redelete` entry — an entry that graduated in an *earlier* round, not the current one — up to `cas_gc_round_redelete_budget`; a `DELETE` only when the body is present at the condemned token | | Successful lease `CAS gc/state` | 1 | | Commit `CAS gc/state` | 1 | @@ -1024,12 +1027,12 @@ checksum disagrees with its seal). | Command | Effect | |---|---| -| `SYSTEM CAS GC RUN []` | One synchronous round execution on the contacted node; only the lease holder makes progress. The disk is optional: without it every content-addressed disk on the node runs one round. It runs even while the scheduler is stopped | +| `SYSTEM CAS GC RUN []` | One synchronous round execution on the contacted node; only the lease holder (the GC leader, `is_leader = 1` in `system.cas_mounts`) makes progress, and on any other node the command does nothing. The disk is optional: without it every content-addressed disk on the node runs one round. It runs even while the scheduler is stopped | | `SYSTEM CAS GC STOP ` / `SYSTEM CAS GC START ` | Stop or resume future background rounds on that disk's scheduler, preserving its identity. The disk is required | | `SYSTEM CAS GC REBUILD [FORCE] ` (`clickhouse-disks cas-gc-rebuild`) | Fail-closed disaster-recovery path for a lost or corrupt `GC` baseline — the `CORRUPTED_DATA` errors that name it in their message (missing adopted seal, snapshot without a surviving log, cursor/apply mismatch). An `ABORTED` commit conflict or a `LOGICAL_ERROR` delete marker is not a reason to rebuild. It deliberately over-protects — it prefers bounded leaks over risking an under-count — and cannot delete live data directly: deletions it produces still flow through the normal round's condemn, graduate, exact-token path. The disk is required | | `clickhouse-disks cas-gc-dryrun` | Opens the disk read-only, constructs a non-leader `GC`, and prints what would be deleted with a reason per entry. Write-free, resolves runs through the seal's references. Documented caveat: it does not fold new owner events, so away from quiescence it can **over-report** — the subset guarantee holds only at quiescence, and its output must never feed a real delete | -`SYSTEM CAS DROP POOL MEMBER '' FROM DISK ''` — permanent removal of a dead +`SYSTEM CAS DROP POOL MEMBER '' FROM DISK ''` — removal of a dead replica, distinct from ordinary `GC` — is covered on the [mounts-and-leases page](/antalya/cas/architecture/mounts-and-leases#mount-lifecycle). `SYSTEM CAS FSCK ` (`clickhouse-disks cas-fsck`) and its `dangling`/`unreachable` vocabulary diff --git a/docs/en/antalya/cas/architecture/mounts-and-leases.md b/docs/en/antalya/cas/architecture/mounts-and-leases.md index 71bd26132d2f5..0e8af640a0753 100644 --- a/docs/en/antalya/cas/architecture/mounts-and-leases.md +++ b/docs/en/antalya/cas/architecture/mounts-and-leases.md @@ -20,8 +20,8 @@ otherwise regenerate the same `ServerUUID` from a wiped local state directory, w silently steal an existing identity. Validation (`validateServerRootId`, `Pool/CasServerRoot.h`) is fail-closed `BAD_ARGUMENTS`, no -sanitizing fallback: non-empty, at most 255 bytes, no empty/`.`/`..` path segment, no `_files` or -`_manifests` segment. +sanitizing fallback: non-empty, at most 255 bytes, no `/`, not `.` or `..`, not `_files` or +`_manifests`. It roots four subtrees and owns catalog names at or below ``: @@ -38,33 +38,83 @@ dedup work. Ref/namespace keys are also deliberately opaque and do not embed the Each replica sharing a backend endpoint must use a distinct `cas_server_root_id`; omitting the setting is a startup error. +The id may not contain `/`, and this holds for an id that already owns its owner object too: decommission +and the staging sweep select a member's objects by key prefix, so a member `a/b` would be drained by a +decommission of `a`. A server whose id contains `/` refuses to open the disk; see +[recreating a member whose id contains `/`](/antalya/cas/operations/migration#recreate-id-with-slash). + ## The owner claim {#owner-claim} -`claimOwnerOrThrow` binds `server_root_id` ↔ `server_uuid` **permanently**. The owner object is never deleted -and never reassigned — decommission only tombstones it in place. +`claimOwnerOrThrow` binds `server_root_id` to one `server_uuid` per owner life. The owner object is never +deleted. Decommission tombstones it in place; a server start over a verified, drained retirement takes it over +at the next life ([rejoin](#rejoin)). | Observed at `gc/server-roots//owner` | Action | |---|---| | present, same `server_uuid`, not tombstoned | proceed | -| present, `retired_at_ms` set | `CORRUPTED_DATA` — explicitly decommissioned, refuses to resume | -| present, different `server_uuid` | `CORRUPTED_DATA` — names the regenerated-uuid-file cause | +| present, `retired_at_ms` set, `retirement_verified` | a server start rejoins when its preconditions hold ([below](#rejoin)); a self-remount stops in `IdentityLost`; the decommission admin accepts its victim's tombstone | +| present, `retired_at_ms` set, not verified | `CORRUPTED_DATA` for a server start: the decommission is not finished; a self-remount stops in `IdentityLost`; the decommission admin accepts it | +| present, different `server_uuid`, not tombstoned | `CORRUPTED_DATA` — names the regenerated-uuid-file cause and, for a dead owner, `SYSTEM CAS DROP POOL MEMBER` | | absent, subtree provably empty | `putIfAbsent` the owner (claim) | | absent, subtree non-empty | `CORRUPTED_DATA` — identity lost over existing data | | lost the `putIfAbsent` race | re-read; equal uuid proceeds, else `CORRUPTED_DATA` | +The owner carries `owner_life`, a counter every owner write sets to the read value + 1 by `If-Match`, so +two owner writes never repeat their bytes and a delayed `If-Match` against an older incarnation cannot +land. A process records the life it started under; after every mount claim (start, self-remount, +decommission) it reads the owner once more and requires its own `server_uuid` and that life +(`recheckOwnerAfterClaim`); otherwise it fences its own mount body and fails, so a successor does not wait +the refused lease out. A self-remount also reads the owner before it claims: a different `server_uuid`, a +retirement or another life moves the pool to `IdentityLost`, at the latest on the next attempt, instead of +retrying. The retirement counts on its own: a tombstone written by 26.6.4 drops `owner_life`. +`retirement_verified` marks a decommission whose claim, tombstone and farewell all held. + "Provably empty" requires both an authoritative decoded catalog naming no life owned by `server_root_id` and a 1-key `LIST` probe finding nothing under `cas/manifests//` or `roots//`. Two failure modes this closes: -- A **second server with a different `server_uuid`** is refused at this gate and can never take - over, regardless of lease expiry. +- A **second server with a different `server_uuid`** is refused at this gate, regardless of lease expiry, + and takes over only by a rejoin after the id's decommission is verified and drained. - A **same-uuid live twin** (two processes sharing one uuid file and `server_root_id`) is caught separately, by the mount claim's token-stability observation, and aborts with an operator-facing message rather than corrupting the pool — this is the default behavior, with `cas_unsafe_remount_no_delay` off. With it on, a same-uuid claim over such a slot reclaims at once instead of observing (see `cas_unsafe_remount_no_delay` in the configuration reference). +### Rejoin at start {#rejoin} + +A server whose `server_root_id` was decommissioned rejoins on its next start, with the same or a new +`server_uuid`. There is no command. It comes back empty: the decommission deleted the data of the +previous life. + +`claimOwnerOrThrow` rejoins only from `Pool::open`, never from a self-remount, and only when: + +- the process has a non-zero `server_uuid` (`clickhouse disks` without a uuid file never rejoins and is refused); +- `epoch` and `mount` exist (checked first: an id retired by 26.6.4 has neither); +- the owner carries `retirement_verified`; +- the slot body is terminal (clean farewell or `gc_fenced`); +- the subtree is empty, after deleting the old life's access-probe key `roots//clickhouse_access_check_`. + +Then a new `server_uuid` first rewrites the terminal `mount` body under its own uuid, and the owner becomes +`{server_uuid, owner_life + 1}` by `If-Match`. A lost `If-Match` re-reads the owner and decides again. The +kept counter gives the new life an epoch above every earlier one. Each of the two writes emits a `MountClaim` +event, `rejoin_mount_rewrite` and `rejoin_owner`, with the retired and the new `server_uuid` and life. A process +of the old life that is still running stops in `IdentityLost` at its next remount. + +| Refusal | Code | Cause | +|---|---|---| +| `no server uuid` | `CORRUPTED_DATA` | the process has no server uuid | +| `cannot rejoin` | `CORRUPTED_DATA` | `epoch` or `mount` is missing: the id was retired by 26.6.4, or the object was deleted by hand | +| `decommission is not finished` | `CORRUPTED_DATA` | the tombstone is not verified; `system.cas_mounts` shows the id as `retired` with `retirement_verified = 0` once its slot is terminal | +| `still running` | `ABORTED` | a decommission run holds the slot, or the lease of a killed run or an interrupted restart has not lapsed and been fenced by GC (up to one GC round after the lease threshold) | +| `previous life remains` | `ABORTED` | the message names the first part left: catalog rows (with the first namespaces and their state), `cas/manifests//` or `roots//` | +| `kept changing` | `ABORTED` | the owner or the slot changed under every rejoin attempt of this start; restart | + +For `previous life remains`, catalog rows and manifests are removed by GC over several rounds: run `SYSTEM CAS GC RUN ''` on the GC leader +(`is_leader = 1` in `system.cas_mounts`) and restart. A late object under `roots//` is not reclaimed by GC and blocks the rejoin until it is deleted by hand. +The operator rule for rerunning a decommission is in [bringing a member back](/antalya/cas/operations/migration#rejoin-rerun-rule). + ## The mount lease {#mount-lease} One object, `gc/server-roots//mount`, carries **both** the liveness lease and the build @@ -81,14 +131,17 @@ watermark — there is no separate watermark object. `MountLease` fields: `serve after a flat pause and settled by the reissue's own answer (a 2xx) or by the exact `GET` that follows its `412`. The renewer adopts the result only when the complete body, including `write_attempt_id`, equals its immutable request. If the predecessor token is still current, another identical `PUT` - may follow bounded backoff. A same-pair twin, GC-fenced body, successor, foreign holder, or absent + follows after a pause. A same-pair twin, GC-fenced body, successor, foreign holder, or absent body is never treated as this renewal. -- **Absolute deadline.** Renewal uses `CLOCK_BOOTTIME`, not `CLOCK_MONOTONIC`, so a VM resumed from - suspend correctly observes itself expired. Its absolute deadline is the minimum of the existing - request-operation budget and the last confirmed lease deadline minus the safety margin. The - controller checks that one attempt envelope still fits before each backend `PUT` or resolving - `GET`, after each interruptible backoff, and before accepting success. A retry, `GET`, response - timestamp, or wall-clock step never extends authority. +- **Deadline from the start.** Renewal uses `CLOCK_BOOTTIME`, not `CLOCK_MONOTONIC`, so a VM resumed + from suspend correctly observes itself expired. A confirmed renewal sets the deadline to its own + start plus the lease; a retry, `GET`, response timestamp, or wall-clock step never moves it. The + background renewal has no deadline of its own: it retries one body every 0.8–1.2 s (measured from + the start of the previous attempt) until the store answers about the slot, or until a stop, a + remount park request, a lost fence or a pool leaving `Live` ends it. A + refusal (`AccessDenied`, `MalformedXML`, …) sends the same body again one renewal period later. + The bounded renewals — the startup re-anchor, the remount re-anchor, and the background renewal of a + pool opened for decommission — stay bounded by the last confirmed lease minus the safety margin. - **Cadence.** The runtime normally starts a logical renewal every `cas_mount_renew_period_ms` (default 10 s), with TTL `cas_mount_lease_ttl_ms` (default 30 s, TTL/3 renewal ratio). The next beat is anchored at the committed body's pre-I/O BOOTTIME start. A slow recovery therefore causes an immediate @@ -103,49 +156,82 @@ watermark — there is no separate watermark object. `MountLease` fields: `serve **Losing the lease is neither read-only mode nor a process abort.** `MountLeaseRenewer` is a synchronous durable-slot state machine. A committed result advances its token, sequence, confirmed -BOOTTIME deadline, and cadence anchor. Any admitted deterministic failure, confirmed conflict, or -ambiguity left at the deadline/attempt limit moves it to `RenewalTerminal`; it cannot mint another -body or publish a clean farewell. Owner cancellation before any request is the only -`NotAttempted` result and leaves clean release possible. Cancellation after a request was sent is -terminal because that request may still land. +BOOTTIME deadline, and cadence anchor. A deterministic failure, a confirmed conflict, or, for a +bounded renewal, ambiguity left at its deadline moves it to `RenewalTerminal`; it cannot mint another +body or publish a clean farewell. Owner cancellation before any request, or between two calls of the +background renewal after a refusal or after a read that found the slot unchanged, is `NotAttempted` and +leaves clean release possible. Cancellation after a request of the current call was sent, or while the +reads after a refused precondition repeat, is terminal: that request may still land, or the slot may +have moved. + +**An expired lease is a pause.** While the store does not answer, writes stop being admitted once the +remaining lease cannot cover one write: `2 × envelope + safety_margin` before the deadline. From that +moment `system.cas_mounts` shows `lifecycle = 'not_live'` and `lifecycle_reason = 'lease_expired'`, +and the server logs one `WARNING`, at the next renewal attempt, that ends with the last renewal error: +`CAS mount lease of '' under writer_epoch has too little time left for a write`. +Nothing is remounted. A confirmed renewal resumes writes under the same `writer_epoch` only if its start +plus the lease leaves more than `2 × envelope + safety_margin`; a renewal confirmed later than that +leaves writes paused, and the next one starts at once. A clean shutdown during a pause skips the +farewell, which has no lease room left, and leaves the slot for GC to observe. Such a late renewal stores an `expires_at_ms` already in the past, and GC still does +not fence the slot for it: GC fences only a token it watched unchanged for its whole threshold. After the renewer call returns, `CasMountRuntime` consumes the result. A terminal result trips the local fence (latches `lost`, bumps the fence generation, moves the in-process runtime to `TransientNotLive`) and latches one self-remount generation. A confirmed foreign/successor or same-pair conflict remains a typed fail-closed error; it is never adopted. A real fence still costs only an epoch: recovery reclaims with a fresh one, bounded at three whole-chain attempts. This is the -general CAS posture: doubt about the source fails closed, while transport ambiguity may retry only -inside authority already proved by the last confirmed lease. +general CAS posture: doubt about the source fails closed, while transport ambiguity is retried until a +definitive answer and writes are admitted only inside the last confirmed lease. GC's own view of a dead server is symmetric and clock-skew-immune: a slot becomes fence-eligible only after the leader observes the *same* renewal token hold stable, on its own monotonic clock, -for `TTL + floor(TTL/20) + period` — close to, but not identical to, the threshold a re-mounting -server uses to wait out a predecessor, which observes `TTL + floor(TTL/20) + max(1, -floor(period/2))`. Both thresholds are evaluated purely on the observer's own clock and its own -configured `TTL`/`period`; nothing about the writer's timing travels on the wire. The stamped -`expires_at_ms` never participates in either decision — it is a writer-stamped diagnostic used by -`system.cas_mounts` and by the non-authoritative decommission epoch-recovery precheck, never an -authorization; local fencing is derived instead from the confirmed request's pre-I/O `BOOTTIME` +for `L + floor(L/20) + period`. `L` is the longer of the leader's own `cas_mount_lease_ttl_ms` and the +lease the slot's body declares (`expires_at_ms − started_at_ms`). A body without stamps, or with a +non-positive interval (a farewell), declares nothing. A re-mounting server waits out a predecessor the +same way with the step `max(1, floor(period/2))`. `period` is always the observer's own. The two stamps +are compared only with each other, never with the observer's clock. `expires_at_ms` alone is a +writer-stamped diagnostic used by `system.cas_mounts`. Local fencing is derived from the confirmed request's pre-I/O `BOOTTIME` anchor plus the TTL, and wall-clock `now` stays audit-only. -Every server sharing a pool must therefore run the identical `cas_mount_lease_ttl_ms` and -`cas_mount_renew_period_ms`: a member or GC leader configured with a shorter threshold than its -peers can fence out a healthy peer whose token-update gap merely exceeds that shorter threshold — -a peer renewing frequently stays live, one that missed a renewal does not. Change these values only -with every member of the pool stopped; a graceful restart removes only that member's own startup -observation and does not make mixed thresholds safe. With the defaults (TTL 30 s, period 10 s, -margin 2 s), `TTL − margin − period − 2 × envelope = 4 s` is the scheduling-lateness budget before +`cas_mount_lease_ttl_ms` is local to each server. It is how long this server may write after its last +confirmed renewal, and every other server waits at least this long before fencing or reclaiming its +slot. A configured value above 10 minutes (600000 ms) is clamped to 10 minutes with a warning at startup, so no holder declares a longer lease. Mixed values are safe, and so is changing the value one server at a time in any order. A longer +lease on one server delays recovery after that server's crash, and the GC leader logs one `WARNING` per +slot whose declared lease differs from its own. A server built without this rule waits only its own +lease: keep the values equal until every server of the pool runs a build with the rule. + +With the defaults (TTL 30 s, period 10 s, margin 2 s), `TTL − margin − period − 2 × envelope = 4 s` is the scheduling-lateness budget before the first renewal attempt of a period can begin, where `envelope = attempt_timeout + 2 × cap` and `cap` is `attempt_timeout` when the disk's `connect_timeout_ms` is `0`, else `min(connect_timeout_ms, attempt_timeout)` (7 s with defaults). +### Cloning a running server is unsupported {#clone-unsupported} + +Do not start a second server from a copy of a running server's data directory or volume, or restore one +snapshot of a running VM twice. The copy carries the `uuid` file, so both processes claim one mount slot +under one `server_uuid`. + +- A second start that finds the first one's slot is refused as `LiveDoubleStart` (same `server_uuid`, no + certificate of death). +- A clone of a process that is already running is worse. It inherits the same `server_uuid`, `writer_epoch` + and lease state, so both processes renew one slot as one incarnation. The lease is not designed to tell them + apart and cannot be relied on to fence the clone. + +To add a server, give it its own `cas_server_root_id` and a fresh `uuid` file. + ## The two monotone counters {#counters} | Counter | Storage | Scope | Protects against | |---|---|---|---| -| `writer_epoch` | durable, `gc/server-roots//epoch` (`ServerEpoch::next_writer_epoch`, CAS-bumped by `allocateWriterEpoch`) | across crashes and restarts | a same-`(uuid, epoch)` twin: a present mount under a normal claim attempt is `CORRUPTED_DATA` | +| `writer_epoch` | durable, `gc/server-roots//epoch` (`ServerEpoch::next_writer_epoch`, CAS-bumped by `allocateWriterEpoch`); never deleted: a retired `server_root_id` keeps its counter | across crashes and restarts | a same-`(uuid, epoch)` twin: a present mount under a normal claim attempt is `CORRUPTED_DATA` | | `build_seq` | in-memory only, `CasMountRuntime::next_build_seq`, reset to 1 on every process start | one process incarnation | orders builds *within* an epoch; combined with `writer_epoch` it gives GC a total order | +Each bump of the `epoch` counter also writes a fresh random 128-bit `allocation_nonce`. When a +counter write's outcome is unclear and the counter has since moved, the allocator counts the write +as landed only if the stored bytes match its own exactly. Two starts of the same `server_uuid` running this allocator that bump +to the same value write different nonces, so they never both adopt one `writer_epoch`. Allocators +of 26.6.4 write no nonce, so two of them can still collide. 26.6.4 readers skip the field. + The absent-epoch branch of `allocateWriterEpoch` is deliberately paranoid: absent with a non-empty subtree is `CORRUPTED_DATA` (reset hazard); absent with an empty subtree decides by an authoritative probe, never by plain-`get` absence, because a transport fault must not be flattened @@ -165,17 +251,21 @@ a `MountClaimResult::Kind` together with a `MountPriorState` describing which ce | `Kind` | Meaning | |---|---| -| `Claimed` | fresh claim (absent slot), same-`(uuid, epoch)` refresh, or a certified reclaim | -| `LiveDoubleStart` | same `server_uuid`, different `writer_epoch`, and no certificate of death yet — a live twin, wait it out | -| `ForeignOwner` | different `server_uuid` — refused unconditionally | +| `Claimed` | fresh claim (absent slot), or a certified reclaim of an older `writer_epoch` | +| `LiveDoubleStart` | same `server_uuid` with an older `writer_epoch` and no certificate of death yet — a live twin, wait it out; or the same `(uuid, epoch)` already in the slot — two starts got one epoch, refused at once | +| `ForeignOwner` | different `server_uuid` — refused, except a terminal body below the claimer's epoch when a fresh owner read shows the claimer's uuid and life (a server start or self-remount after a rejoin) | | `FencedSelf` | same `(uuid, epoch)`, but `gc_fenced` — terminal for *this* epoch; the caller must mint a fresh one | +A claim never installs a `writer_epoch` at or below the one in the slot. When the slot holds a newer +`writer_epoch` of the same `server_uuid`, the claim fails with `ABORTED` and writes nothing: another +start or a decommission of this `server_root_id` allocated after this one. + | `MountPriorState` | Certificate that justified the reclaim | |---|---| -| `None` | no reclaim needed (fresh claim or same-epoch refresh) | +| `None` | no reclaim needed (fresh claim) | | `Clean` | the predecessor's own graceful farewell (`min_active_build_sequence == UINT64_MAX`) | | `Fenced` | GC's own threshold-gated fence-out (`gc_fenced`) | -| `UncleanObserved` | this claimant's own token-stability observation held for the full `TTL + drift` window | +| `UncleanObserved` | this claimant's own token-stability observation held for the full observation threshold (the longer of the two leases, plus drift and step) | | `UncleanUnsafe` | the operator's explicit `cas_unsafe_remount_no_delay` authorization carried the slot's exact token — not a certificate of death | ## Behavioral mount-slot model {#mount-state-machines} @@ -190,12 +280,14 @@ stateDiagram-v2 Live --> Live: renewer beat, putOverwrite seq+1 Live --> Fenced: GC observes a stable token past threshold, gc_fenced=1, body preserved Live --> Terminated: certified drain, terminal farewell (expires_at=now, min_active_build_sequence=MAX) - Fenced --> Live: same-uuid claim with a fresh writer_epoch, instant reclaim - Terminated --> Live: same-uuid claim with a fresh writer_epoch, instant reclaim + Fenced --> Live: same-uuid claim with a higher writer_epoch, instant reclaim + Terminated --> Live: same-uuid claim with a higher writer_epoch, instant reclaim Live --> Live: same-uuid claim, proven-dead token via UncleanObserved Live --> Live: same-uuid claim under cas_unsafe_remount_no_delay, no observation Fenced --> Fenced: same uuid and epoch claim, FencedSelf, no write - Live --> Absent: decommission tail, mount then epoch then owner tombstone + Terminated --> Terminated: decommission farewell, owner tombstoned and verified + Terminated --> Live: rejoin of a verified, drained retirement, any server_uuid + Fenced --> Live: rejoin of a verified, drained retirement, any server_uuid Terminated --> [*] ``` @@ -208,7 +300,7 @@ stateDiagram-v2 Live --> TransientNotLive: renewal failure, tripMountLost, lost=true TransientNotLive --> Live: self-remount succeeds with a fresh epoch TransientNotLive --> TransientNotLive: probe inconclusive, retry with backoff - TransientNotLive --> IdentityLost: pool meta and owner both authoritatively absent + TransientNotLive --> IdentityLost: pool meta and owner both absent, or the owner at another life TransientNotLive --> VanishedReplaced: foreign pool_id observed Live --> VanishedForgotten: SYSTEM CAS FORGET IdentityLost --> [*] @@ -254,26 +346,51 @@ while a stale conditional request from the predecessor is still in flight. and waiting out the token-stability observation, or the GC leader fencing the slot first, after which any reclaim is instant. -**Permanent removal** of a dead replica (`Cas::decommissionPoolMember`, driven by -`SYSTEM CAS DROP POOL MEMBER '' FROM DISK ''`) claims the victim's mount slot -as an administrative writer with a no-wait policy (refuses immediately if the member is alive), -drops every ref-bearing namespace, sweeps manifest debris before the slot (deleting the mount -removes the watermark authority), drains staging and roots, then — only with zero warnings — -retires in order: mount, epoch, a final liveness re-check, owner tombstone. +**Removal** of a dead replica (`Cas::decommissionPoolMember`, `SYSTEM CAS DROP POOL MEMBER '' FROM DISK ''`) +keeps `owner`, `epoch` and `mount` and removes only data, in six steps: + +1. Claim the victim's slot as an administrative writer with a no-wait policy. The claim needs a terminal + slot (`terminated` or `fenced`); an `expired` slot is refused until the disk's GC leader fences it. A + member without an epoch counter is refused. +2. Tombstone the owner at the next `owner_life`, on the admin's fenced mount plane: no attempt is sent + after the admin's lease deadline, but the store may apply a sent one later. When the outcome is unknown and + the owner does not read back as the tombstone, the run fails without a farewell: the victim restarts only + after GC fences the admin's slot or its own start sees the lease hold still for the observation threshold. + A tombstone applied later than that is the case the + [rerun rule](/antalya/cas/operations/migration#rejoin-rerun-rule) repairs by hand. +3. Confirm the slot still holds the admin's claim. A claim lost before the drain rolls the tombstone back, + unless the owner was already retired before this run. A claim lost during the drain keeps the tombstone: + the drain may have removed data, and the tombstone keeps the member from mounting over it. Every failed + exit between (2) and (5) runs this confirm; a clean drain is checked by the verify in (6). +4. Drop every namespace, sweep manifest debris, drain `staging/` and `roots/`, all on the admin's fenced + mount plane. +5. Write the admin's clean farewell. +6. Read the farewell and the tombstone back and set `retirement_verified`. + +`member_retired = 1` only when the owner carries a verified retirement. An owner that moved on to a later, +unretired life after the tombstone (a rejoin, or another run's rollback) reports `member_retired = 0` with +the warning `the owner moved on to life N`. Catalog rows GC still has to delete are reported as +`namespaces_pending_gc`, not as a failure. The operator reruns only after `member_retired = 0`, under the +[rerun rule](/antalya/cas/operations/migration#rejoin-rerun-rule); a rerun refused as alive or contended +waits until GC fences the previous run's slot: about one mount lease TTL plus one GC round, with a GC leader +running. A later start under that `server_root_id` rejoins it as an empty member once the retirement is +verified and drained ([rejoin](#rejoin)). ## `system.cas_mounts` {#mounts-table} A read-only view of the same heartbeat-floor computation GC uses: one `LIST` of -`gc/server-roots/` plus one `GET` per slot, zero writes, per-row fail-open (an undecodable body +`gc/server-roots/` plus one `GET` per slot, plus one owner `GET` per terminated or fenced slot, zero writes, per-row fail-open (an undecodable body, or an undecodable owner of a terminated or fenced slot, becomes `state = 'corrupt'`, never an exception). Shows every `server_root_id` in the pool, including peers. | Column | Notes | |---|---| | `disk`, `server_root_id`, `server_uuid`, `hostname`, `process_id` | identity | | `writer_epoch`, `renewal_sequence`, `started_at`, `expires_at`, `min_active_build_sequence`, `gc_fenced` | lease state (`DateTime64(3)` columns; the millisecond-integer field names live only in the internal `MountLease` struct and the on-disk body) | -| `state` | one of `live`, `expired`, `terminated`, `fenced`, `corrupt` | +| `state` | one of `live`, `expired`, `terminated`, `fenced`, `retired`, `corrupt`; `retired` = terminated or fenced with a tombstoned owner (one more `GET` per terminated or fenced row) | | `is_leader`, `pending_reclaim`, `last_success_age_seconds`, `wedged_namespace_count` | GC health, process-local; **`NULL` on every peer row** — a process-local fact must never be stamped onto another server's row | | `lifecycle`, `lifecycle_reason`, `lifecycle_detail`, `lifecycle_since` | the SQL surface for the in-process `PoolLifecycle` runtime above: `lifecycle` is one of `live`, `not_live`, `identity_lost`, `vanished`, `constructing`, `shutdown`; `lifecycle_reason` distinguishes `replaced` from `forgotten` for a `vanished` disk; `lifecycle_detail` carries the full diagnosis text; `lifecycle_since` is when the current non-live state began (`NULL` while live) | +| `renewal_in_progress`, `renewal_attempts`, `last_renewal_error` | the running (or last) lease renewal: conditional `PUT`s sent so far and the last failed request, updated per attempt; **`NULL` on every peer row** | +| `retirement_verified` | `1` on a `retired` row whose decommission verified the retirement, else `0`; a rejoin needs `1`, a rerun is for `0` only | The lifecycle snapshot is I/O-free and ungated, so a not-live, never-started, or vanished disk still produces a row instead of silently disappearing from the table. diff --git a/docs/en/antalya/cas/architecture/part-lifecycle.md b/docs/en/antalya/cas/architecture/part-lifecycle.md index f5ad05dc5f0fb..a762ca6c7eb70 100644 --- a/docs/en/antalya/cas/architecture/part-lifecycle.md +++ b/docs/en/antalya/cas/architecture/part-lifecycle.md @@ -1,5 +1,5 @@ --- -description: 'The part-add protocol from local build through blob upload to promote, its nine crash points and their cleaners, and how each MergeTree operation maps onto it.' +description: 'The part-add protocol from local build through blob upload to promote, its crash points and their cleaners, and how each MergeTree operation maps onto it.' sidebar_label: 'Part lifecycle' sidebar_position: 6 slug: /antalya/cas/architecture/part-lifecycle @@ -90,7 +90,7 @@ calling thread only submits and joins, never occupies a pool slot, so a pool of to a correct serial run and can never deadlock. The contract is merge-nothing: if any task threw, nothing is merged and the first error in dispatch order is rethrown. Results are folded into the dependency set on the owning thread, into a copy, committed by one no-throw swap. Every physical -task starts with blob `HEAD`; a present non-condemned body or a completed publication yields an +task starts with blob `HEAD`; a present body with a `Clean` marker or a completed publication yields an explicit `Materialized` proof. A trusted source manifest instead yields `TrustedManifest` without blob I/O. Pool size is the server setting `cas_blob_upload_pool_size` (default 16). @@ -106,20 +106,26 @@ manifest never loses an owner, so it is net zero. ## Crash points and their cleaners {#crash-points} -This table is the single best summary of the design's crash-safety story: every row leaks -something recoverable; no row loses data or leaves a dangling reference. +This table is the single best summary of the design's crash-safety story: no row loses committed +data or leaves a dangling reference. In C6 and C10 the part itself may take effect; C3 and C11 +leave only a precommit binding. | # | Crash window | Left behind | Who cleans it | |---|---|---|---| -| C1 | During staging | Local temp files, or S3 staging objects | Local: unconditional cleanup plus buffer destructor. S3: the mount's own staging sweep at next mount — never deleted on abort | +| C1 | During staging | Local temp files, or S3 staging objects | Local: unconditional cleanup plus buffer destructor. S3: the transaction deletes it on commit or abort (best-effort); after a crash, the next mount of the same `server_root_id` sweeps it, or `SYSTEM CAS DROP POOL MEMBER` drains it | | C2 | After `stageManifest`, before `precommitAdd` | An unreferenced manifest body | Writer's best-effort exact-token delete; durable backstop is the orphan-manifest sweep | | C3 | `precommitAdd` returned Unresolved | A possibly-live precommit binding | Intent recorded pre-append; `abandon` appends the exact removal, tolerating absence. The body is never writer-deleted | | C4 | Between `precommitAdd` and `promote` | A live precommit plus uploaded blobs | No resume path exists. Removed by `abandon`, else by a fenced successor's stale-precommit sweep | | C5 | Mid blob fan-out | Already-uploaded blobs | Nothing merged; blobs become `GC`-reclaimable debris; the part is not published | -| C6 | `promote` append Unresolved | The ref may or may not be committed | Commit state Uncertain — the relink layer maps this to "retry the whole fetch", never to a byte fetch | +| C6 | `promote` append Unresolved | The ref may or may not be committed | Commit state Uncertain — the relink layer maps this to "retry the whole fetch", never to a byte fetch. The client sees `outcome is unknown` | | C7 | A later part throws after earlier parts published | A partial multi-part commit | Precise rollback: drop only the refs this call created, matching the exact manifest — never clobbers a concurrent writer's repoint | | C8 | Transaction destroyed uncommitted | Open builds | Destructor abandons every build | | C9 | Namespace dropped mid-build | — | One atomic flag; every further op fails closed at the alive check | +| C10 | `promote` ref-log `PUT` committed, then the `_ckpt` frontier publish failed | A durable promotion this server has not installed; the ref lane needs recovery | The next ref-log recovery installs it. The in-memory part was already rolled back, and part cleanup later drops the ref. A restart before that drop brings the part back on `MergeTree`; on `ReplicatedMergeTree` the part has no Keeper entry and startup detaches it as `ignored_`. The client sees `is durable but was not acknowledged`, and the message says the write may still take effect | +| C11 | `precommitAdd` ref-log `PUT` committed, then the `_ckpt` frontier publish failed | A durable precommit binding this server has not installed | The next ref-log recovery installs the binding; the part is never published. `abandon`, else the stale-precommit sweep, removes it. The client sees `is durable but was not acknowledged` with `the part is never published` | + +What each client-visible message means, and when a retry is safe, is in +[CAS commit errors](/antalya/cas/operations/troubleshooting#cas-commit-errors). ## The repoint {#repoint} diff --git a/docs/en/antalya/cas/bucket-requirements.md b/docs/en/antalya/cas/bucket-requirements.md index ea4bfe0dbafed..a42a6714998a4 100644 --- a/docs/en/antalya/cas/bucket-requirements.md +++ b/docs/en/antalya/cas/bucket-requirements.md @@ -45,6 +45,12 @@ Because that check is part of the mount battery, `skip_access_check = true` is r generation-token disk. Mount the disk read-only if you need to start before the access check can pass. +**Incomplete multipart uploads.** With `cas_staging_backend = s3`, a process killed during a +multipart upload leaves an incomplete upload under `staging//`. Object `LIST` does +not show it, so neither the transaction's own delete nor the mount-start sweep can remove it, and its +parts stay billed. Add an `AbortIncompleteMultipartUpload` lifecycle rule on the bucket (one day is +enough: a staging upload completes within its own transaction or never). + ## Soft delete is an operator precondition {#soft-delete} Object **soft delete must be disabled** on a `CAS` bucket, and unlike versioning this one is *not* diff --git a/docs/en/antalya/cas/configuration.md b/docs/en/antalya/cas/configuration.md index 80e09cdb71b13..80d150de2652d 100644 --- a/docs/en/antalya/cas/configuration.md +++ b/docs/en/antalya/cas/configuration.md @@ -89,7 +89,7 @@ entirely before release. Treat this table as a snapshot of the current build, no | Setting | Default | Description | |---|---|---| -| `cas_server_root_id` | — (required) | Explicit layout subtree identity; macros expand as in the `s3` `endpoint`. Anchored in the pool by a write-once owner claim — a colliding identity is refused at mount | +| `cas_server_root_id` | — (required) | Explicit layout subtree identity; macros expand as in the `s3` `endpoint`. Anchored in the pool by a write-once owner claim — a colliding identity is refused at mount. May not contain `/`, also after macro expansion: `SYSTEM CAS DROP POOL MEMBER` selects objects by key prefix, so a member `a/b` would be drained by a decommission of `a`. A disk configured with such an id refuses to open, including one whose id already owns its owner object in the pool; see [recreating a member whose id contains `/`](/antalya/cas/operations/migration#recreate-id-with-slash) | | `cas_scratch_path` | `/disks//cas_scratch/` | Server-local scratch dir for the write-buffer spill; a relative value is anchored to the server data path | | `cas_gc_enabled` | `true` | Run the background GC scheduler on this disk. `false` is a debugging aid, not an operating mode: garbage then accumulates indefinitely and silently — watch `system.cas_gc_log` for round activity if you ever toggle it | | `cas_gc_interval_sec` | `60` | Seconds between background GC rounds (≥ 1) | @@ -97,7 +97,7 @@ entirely before release. Treat this table as a snapshot of the current build, no | `cas_blob_hash_allow_new` | `false` | Explicit opt-in to admit a new hash algorithm into an existing pool. One-way: once admitted, the pool carries both algorithms permanently | | `cas_allow_packed_parts` | `false` | Allow creation of new `Packed` MergeTree parts on this disk. The decision uses the actual destination disk, not every disk in its storage policy. A `type=cache` disk inherits this capability from the wrapped `CAS` disk; set the key in the `metadata_type=cas` disk block. Existing `Packed` parts remain readable when this setting is `false` | | `skip_access_check` | `false` | Skip the boot-time capability probe (start now, fix later). Only the preflight probe is skipped — the conditional-write correctness check still runs on every writable mount. **Not available on a writable generation-token (GCS) disk**, which refuses to mount with it: there, the probe battery is the only proof that a token-exact delete carries its generation precondition. Mount such a disk read-only if you need to defer the check | -| `cas_mount_lease_ttl_ms` | `30000` | Milliseconds for which a mount lease remains valid after a successful claim or renewal (≥ 1). Lower values shorten stale-mount recovery but reduce tolerance for object-storage and scheduling delays | +| `cas_mount_lease_ttl_ms` | `30000` | How long this server may write after its last confirmed renewal; every other server waits at least this long before fencing or reclaiming its slot (≥ 1, at most 600000 — a larger value is clamped to 10 minutes with a warning at startup, so no holder declares a longer lease). Lower values shorten recovery after this server's crash but reduce tolerance for object-storage and scheduling delays | | `cas_mount_renew_period_ms` | `10000` | Milliseconds between background mount-lease renewals (≥ 1). It must leave enough time for two attempt envelopes (a renewal write and its settlement read) and the lease safety margin before the TTL expires: `period + 2 × envelope + margin < TTL` | | `cas_gc_snapshot_generations_to_keep` | `3` | GC snapshot generations retained | | `cas_gc_shards` | `1` | Blob-hash-prefix reducer shards (≥ 1). Recorded in the pool at creation; a mismatching config is refused at mount | @@ -108,10 +108,35 @@ entirely before release. Treat this table as a snapshot of the current build, no | `cas_manifest_decode_cache_bytes` | 128 MiB | Manifest decode cache byte budget (`0` disables) | | `cas_gc_meta_pool_size` | `16` | Bounded pool size for GC per-hash freshness-meta writes | | `cas_gc_io_concurrency` | `16` | Bounded pool size for GC object-storage requests that run in parallel: the fold's read-ahead (checkpoints, ref logs, manifests, zero-candidate HEADs), the orphan-manifest sweep planning reads, the `SYSTEM CAS GC REBUILD` read-ahead, the `pending_deletes` blob `HEAD` + conditional `DELETE` fan-out, and the namespace janitor's batch deletes of dead ref logs and snapshots. Not covered: meta writes (`cas_gc_meta_pool_size`) and all other GC requests, which run on the round thread. `1` runs the covered requests sequentially. `cas_gc_read_concurrency` is rejected without an alias; use `cas_gc_io_concurrency` instead | -| `cas_attempt_timeout_ms` | `5000` | Budget for one HTTP attempt of a writable Native mount's control-plane requests (read, head, list, remove, conditional write), at least 1. Together with the connect cap it forms the attempt envelope (`cas_attempt_timeout_ms + 2 × cap`; the cap is `cas_attempt_timeout_ms` itself when the disk's `connect_timeout_ms` is `0`, else `min(connect_timeout_ms, cas_attempt_timeout_ms)`) that the lease arithmetic reserves: one TCP connect and one TLS handshake under the cap each, send/receive bounded per socket operation by `cas_attempt_timeout_ms`. With background renewal the cadence check requires `cas_mount_renew_period_ms + 2 × envelope + cas_lease_safety_margin_ms < cas_mount_lease_ttl_ms`, which puts an effective ceiling on the frozen connect cap: under the defaults (TTL 30000, period 10000, margin 2000) the envelope must stay under 9000, so a disk `connect_timeout_ms` of 2000 ms or more refuses to open writable — lower the connect timeout or raise the TTL if you hit this | +| `cas_attempt_timeout_ms` | `5000` | Budget for one HTTP attempt of a writable Native mount's control-plane requests (read, head, list, remove, conditional write), at least 1. Together with the connect cap it forms the attempt envelope (`cas_attempt_timeout_ms + 2 × cap`; the cap is `cas_attempt_timeout_ms` itself when the disk's `connect_timeout_ms` is `0`, else `min(connect_timeout_ms, cas_attempt_timeout_ms)`) that the lease arithmetic reserves: one TCP connect and one TLS handshake under the cap each, send/receive bounded per socket operation by `cas_attempt_timeout_ms`. With background renewal the cadence check requires `cas_mount_renew_period_ms + 2 × envelope + cas_lease_safety_margin_ms < cas_mount_lease_ttl_ms`, which puts an effective ceiling on the frozen connect cap: under the defaults (TTL 30000, period 10000, margin 2000) the envelope must stay under 9000, so a disk `connect_timeout_ms` of 2000 ms or more refuses to open writable — lower the connect timeout or raise the TTL if you hit this. Blob body publication does not use this budget: its object-creating request is sent once under the disk's `request_timeout_ms`, with the connect timeout under the same cap | | `cas_lease_safety_margin_ms` | `2000` | Startup-only margin validated against the mount lease TTL: the attempt envelope + `cas_lease_safety_margin_ms` must be strictly less than the mount lease TTL, and `cas_mount_renew_period_ms` + 2 × envelope + `cas_lease_safety_margin_ms` too, or the disk refuses to open writable | | `cas_unsafe_remount_no_delay` | `0` | Reclaim a mount slot that carries this server's own uuid at once after a hard restart, without observing the slot's token for the lease TTL. Unsafe whenever two processes can hold the same `server_uuid` (a copied uuid file, a stalled predecessor). After such a reclaim the predecessor can still start conditional writes until its own cutoff (`confirmed deadline − cas_lease_safety_margin_ms − 2 × envelope`) or until its next renewal meets the token guard, and a request it already sent may still materialize later. That is not a data hazard: ref-log keys carry `(writer_epoch, sequence)` and creates are conditional, so two writers can never commit different bodies to one key, and recovery's epoch seal settles any straggler (recovery fails closed after 64 successive seal-create attempts displaced by newly materializing old-epoch transactions). The exposure is availability, not data. Intended for test stands and deployments that guarantee one process per uuid | -| `cas_staging_backend` | `local` | Blob staging backend (`local` \| `s3`); `s3` is opt-in and requires native same-store copy on writable mount | +| `cas_staging_backend` | `local` | Blob staging backend (`local` \| `s3`); `s3` is opt-in and requires native same-store copy on writable mount. A transaction deletes its staging objects on commit and on abort; mount start removes leftovers of the same `server_root_id`. Neither can see a multipart upload that was never completed, so add an `AbortIncompleteMultipartUpload` lifecycle rule on the bucket to reclaim those | + +Both settings are local. Startup reclaim and GC's fence-out judge liveness by the mount slot's write +token holding stable on the observer's own `CLOCK_BOOTTIME` for `L + floor(L / 20) + step`. `L` is the +longer of the observer's `cas_mount_lease_ttl_ms` and the lease the holder's body declares +(`expires_at_ms − started_at_ms`; a body without stamps or with a non-positive interval declares +nothing). Startup's step is `max(1, floor(cas_mount_renew_period_ms / 2))` and GC's is +`cas_mount_renew_period_ms`, both the observer's own. GC checks only at the start of a round, so a +fence-out can land up to one GC round after that. Mixed values are therefore safe in any order; +a longer lease on one server delays recovery after that server's crash. A server built without this +rule waits only its own lease, so keep the values equal until every server of the pool runs a build +with it. + +A shorter TTL reduces the tolerance for object-storage delays; a shorter renewal period increases it +(renewal starts earlier) at the cost of more background traffic. With the defaults, +`cas_mount_lease_ttl_ms − cas_lease_safety_margin_ms − cas_mount_renew_period_ms − 2 × envelope = +4000` ms is the scheduling-lateness budget before the first renewal attempt of a period can begin, +where `envelope = cas_attempt_timeout_ms + 2 × cap` (7000 ms with defaults) and `cap` is +`cas_attempt_timeout_ms` when the disk's `connect_timeout_ms` is `0`, else +`min(connect_timeout_ms, cas_attempt_timeout_ms)` (1000 ms with defaults); the renewal then keeps +retrying until `confirmed deadline − cas_lease_safety_margin_ms`. + +The `expires_at_ms` stamped into the mount object is a writer-stamped diagnostic used by +`system.cas_mounts`; it never +authorizes a reclaim or a GC fence-out. Local fencing is derived instead from the confirmed +request's pre-I/O `CLOCK_BOOTTIME` anchor plus the TTL. ### Enabling `Packed` parts {#enabling-packed-parts} @@ -154,32 +179,6 @@ Downgrade to a binary without CAS Packed support is not supported in the current procedure. Such an older binary also rejects the unknown `cas_allow_packed_parts` key even when its value is `false`. -All servers sharing a pool must run the same `cas_mount_lease_ttl_ms` and `cas_mount_renew_period_ms`. -Startup reclaim and GC's fence-out both judge liveness by the mount slot's write token holding stable -on the observer's own `CLOCK_BOOTTIME`, using the observer's own threshold — nothing about a writer's -timing travels on the wire. Startup observes `cas_mount_lease_ttl_ms + floor(cas_mount_lease_ttl_ms / -20) + max(1, floor(cas_mount_renew_period_ms / 2))`; GC observes `cas_mount_lease_ttl_ms + -floor(cas_mount_lease_ttl_ms / 20) + cas_mount_renew_period_ms`. A pool member or GC leader -configured with a shorter threshold than its peers can therefore fence out a healthy peer whose -token-update gap exceeds that shorter threshold — a peer renewing frequently stays live, one that -missed a renewal does not. Change these values only with every member of the pool stopped: a -graceful restart removes only that member's own startup observation and does not make mixed -thresholds safe. - -A shorter TTL reduces the tolerance for object-storage delays; a shorter renewal period increases it -(renewal starts earlier) at the cost of more background traffic. With the defaults, -`cas_mount_lease_ttl_ms − cas_lease_safety_margin_ms − cas_mount_renew_period_ms − 2 × envelope = -4000` ms is the scheduling-lateness budget before the first renewal attempt of a period can begin, -where `envelope = cas_attempt_timeout_ms + 2 × cap` (7000 ms with defaults) and `cap` is -`cas_attempt_timeout_ms` when the disk's `connect_timeout_ms` is `0`, else -`min(connect_timeout_ms, cas_attempt_timeout_ms)` (1000 ms with defaults); the renewal then keeps -retrying until `confirmed deadline − cas_lease_safety_margin_ms`. - -The `expires_at_ms` stamped into the mount object is a writer-stamped diagnostic used by -`system.cas_mounts` and by the non-authoritative decommission epoch-recovery precheck; it never -authorizes a reclaim or a GC fence-out. Local fencing is derived instead from the confirmed -request's pre-I/O `CLOCK_BOOTTIME` anchor plus the TTL. - ## Recommended keep-alive settings {#recommended-keep-alive-settings} On a `CAS` disk, set `http_keep_alive_timeout` to `30` and `http_keep_alive_max_requests` to `10000`, diff --git a/docs/en/antalya/cas/operations/debugging.md b/docs/en/antalya/cas/operations/debugging.md index 1da3143023a31..24c79eb2d84dd 100644 --- a/docs/en/antalya/cas/operations/debugging.md +++ b/docs/en/antalya/cas/operations/debugging.md @@ -153,21 +153,22 @@ row followed by a `recovered` one — and the older `unresolved_reason`, `deadli `stop_cause` keys are gone; everything they used to distinguish is now named directly by `classification`. Interpret the sequence as follows: -- `outcome = 'recovered'` means an in-budget renewal landed, in the same epoch. `classification` +- `outcome = 'recovered'` means a renewal landed after a retry or a settling read, in the same epoch. + Writes are re-admitted only if it left more than `2 × envelope + safety_margin` of lease. `classification` says how: `committed_by_read` means an exact `GET` proved a landed request; `committed_after_retry` means a later identical physical `PUT` completed and the response itself proved it. -- `outcome = 'failed'` carries the decisive `classification`: `external_lease_deadline` (the - confirmed lease's own safety margin, not the request policy, ran out first — check object-store - latency or `BOOTTIME` advancement before anything else), `request_deadline` (the ninety-second - request policy exhausted first), `unresolved` (every attempt was ambiguous and never settled by +- `outcome = 'failed'` carries the decisive `classification`: `external_lease_deadline` (a + bounded renewal only: the confirmed lease's own safety margin, not the request policy, ran out first — check object-store + latency or `BOOTTIME` advancement before anything else), `request_deadline` (a bounded + renewal only: the ninety-second request policy exhausted first), `unresolved` (every attempt was ambiguous and never settled by the time the operation gave up), `conflict` (an exact resolve read found another body — a same-pair twin, a GC-fenced body, a successor epoch, or a foreign holder), `cancelled` (a renewal in flight was cancelled by shutdown or a remount park request; expected during graceful shutdown), `fence_or_lifecycle_lost` (another local fence loss or a terminal lifecycle transition - closed admission while the operation was active), `deterministic_failure` (the store's own - answer proved the write never applied), and `vanished` (an exact resolve read proved the mount - slot absent — the pool directory was removed or renamed out of band, or a decommission raced the - renewal). `terminal_unclassified` means the renewal terminated through a path that assigned no + closed admission while the operation was active), `deterministic_failure` (a local + failure, or, for a bounded renewal, a store answer proving the write never applied; + the background renewal sends a refused write again), and `vanished` (an exact resolve read proved the mount + slot absent — the pool directory was removed or renamed out of band, or a 26.6.4 decommission deleted the slot; a decommission of this version keeps it). `terminal_unclassified` means the renewal terminated through a path that assigned no classification; that is a defect to report together with the surrounding rows, not an operator condition. Do not collapse these into a generic timeout — the action differs by classification, and only `external_lease_deadline` and `request_deadline` are about a @@ -215,6 +216,9 @@ available: SYSTEM CAS GC RUN cas; ``` +Run it on the GC leader (`is_leader = 1` in `system.cas_mounts`). On any other node it does nothing and +returns no error. + One row per disk it ran on: `disk`, `acquired_lease`, `deferred`, `round`, `candidates_marked`, `objects_deleted`, `objects_absent`, `objects_replaced`, `objects_spared`, `manifests_deleted`, `entries_condemned`, `entries_graduated`, `entries_redeleted`, `entries_redelete_failed`, `fence_outs`, @@ -249,8 +253,8 @@ SYSTEM CAS FORGET cas; It is an assertion, not a proof of erasure: the disk stays registered and answers further store-class access with a typed error, and a server restart re-registers the name. This is different from -[`SYSTEM CAS DROP POOL MEMBER`](/antalya/cas/operations/migration#decommission), which permanently -retires one pool *member*'s identity across the whole shared pool — `FORGET` only affects this node's +[`SYSTEM CAS DROP POOL MEMBER`](/antalya/cas/operations/migration#decommission), which decommissions +one pool *member* across the whole shared pool — `FORGET` only affects this node's own local view of one disk. ## Offline tools {#offline-tools} diff --git a/docs/en/antalya/cas/operations/migration.md b/docs/en/antalya/cas/operations/migration.md index e8258b96d61d3..0ef2cde6c2af3 100644 --- a/docs/en/antalya/cas/operations/migration.md +++ b/docs/en/antalya/cas/operations/migration.md @@ -1,5 +1,5 @@ --- -description: 'Adding a content-addressed disk to an existing deployment, moving a partition onto it with ALTER TABLE MOVE PARTITION, rolling back, and permanently decommissioning a pool member.' +description: 'Adding a content-addressed disk to an existing deployment, moving a partition onto it with ALTER TABLE MOVE PARTITION, rolling back, and decommissioning a pool member.' sidebar_label: 'Migration' sidebar_position: 1 slug: /antalya/cas/operations/migration @@ -164,17 +164,20 @@ directly (the raw disk, not the cache) was refused with `All parts of partition already on disk 'cas_cache'. (UNKNOWN_DISK)` — a real error message from the run, kept here because it is exactly what an operator sees after guessing the wrong disk name. -## Permanently removing a pool member {#decommission} +## Removing a pool member {#decommission} A `CAS` pool can be shared by several servers (see [`server_root_id`](/antalya/cas/architecture/mounts-and-leases#server-root-id)). -Scaling down — permanently removing a server that will never rejoin the pool — is a distinct, -irreversible operation from an ordinary restart or a temporary outage: it fences the member's -`server_root_id` and reclaims the storage attributable only to it. +Scaling down — removing a server and its data from the pool — is a distinct operation from an ordinary +restart or a temporary outage: it fences the member's current life and reclaims the storage attributable +only to it. The data is gone for good; the `server_root_id` can rejoin later as an empty member +([rejoin](#rejoin)). `SYSTEM CAS DROP POOL MEMBER` claims the victim's mount slot as an administrative writer (refusing -immediately if the member is still alive), drops every table namespace the member owned, sweeps -manifest debris, drains its staging and mountpoint objects, and — only once every drain is -confirmed — retires the mount slot itself. It emits ordinary ref-edge deltas rather than a GC +immediately if the member is still alive), tombstones its owner, drops every table namespace the member +owned, drains its debris and marks the retirement verified; the steps are in +[mount, unmount, crash](/antalya/cas/architecture/mounts-and-leases#mount-lifecycle). The member's control +objects (`owner`, `epoch`, `mount`) are kept: the epoch counter keeps epochs from being reused when the +`server_root_id` rejoins. It emits ordinary ref-edge deltas rather than a GC transition: it does not synchronously reclaim shared blob content, it only makes the now-unreferenced blobs eligible for an ordinary GC round to reclaim later. @@ -192,22 +195,105 @@ clickhouse-disks -C config.xml --disk cas cas-drop-member 'replica-2' ``` The command returns one row (or, offline, one line per field) with `namespaces_removed`, -`namespaces_already_removed`, `committed_refs_removed`, `precommits_removed`, -`manifest_debris_removed`, `staging_objects_removed`, `mountpoint_objects_removed`, and -`slot_removed`. It is resumable: a rerun skips namespaces already marked removed and reports them -under `namespaces_already_removed` rather than redoing the work. A per-object drain failure is -recorded as a `warning` rather than raised as an exception, leaving the slot terminated but not -fully drained so a later invocation can resume; a non-empty `warnings` means exactly that, and the -mount slot stays in place as a resume anchor rather than being fully retired. - -**Preconditions.** Confirm the member is actually and permanently dead before running this: the -operation fences that `server_root_id` out even if the server comes back online, and it deletes -namespace and drain state that cannot be recovered. Check `system.cas_mounts` for the member's -`state` and `last_success_age_seconds` first — a `live` row, or one with a recent lease renewal, -means the member is not a decommission candidate yet. - -**Verification.** After the command reports `slot_removed = true` with no warnings, the member's -`server_root_id` no longer appears as a row in `system.cas_mounts` on any peer, and a subsequent -`SYSTEM CAS GC RUN` on the pool will no longer wait on or fence its heartbeat. See +`namespaces_already_removed`, `committed_refs_removed`, `precommits_removed`, `manifest_debris_removed`, +`staging_objects_removed`, `mountpoint_objects_removed`, `member_retired`, `warnings` and +`namespaces_pending_gc`; the SQL row ends with `namespaces_pending_gc`. It is resumable: a rerun skips namespaces already marked removed and reports them +under `namespaces_already_removed` rather than redoing the work. A per-object drain failure is recorded as +a `warning`; the owner keeps an unverified tombstone and the next run resumes over it. So does a run that +loses its claim during the drain; only a claim lost before the drain rolls the tombstone back. `member_retired = 1` +means the owner carries a verified retirement; rows still listed under `namespaces_pending_gc` are deleted +by GC and need no rerun. See [troubleshooting](/antalya/cas/operations/troubleshooting) for +`member_retired = 0`. + +**Preconditions.** Confirm the member is actually dead before running this: the +operation deletes namespace and drain state that cannot be recovered, and a server that comes back under +that id after the run starts empty. The command claims the slot without waiting +and trusts no wall clock, so it accepts only a terminal slot in `system.cas_mounts`: +`state = 'terminated'` (clean stop) or `state = 'fenced'` (GC fenced the lease). A `live` or `expired` +row is refused with `alive or contended`; a hard-killed member stays `expired` until the disk's GC leader fences it at the start of a GC round, once it has seen the lease unchanged for about one +`cas_mount_lease_ttl_ms` plus one `cas_mount_renew_period_ms`; the wait can be one GC round longer. `SYSTEM CAS GC RUN` on the leader +(`is_leader = 1` in `system.cas_mounts`) can trigger the fence, but the run that fences must come at +least that long after the first run that saw the lease; two back-to-back runs do not fence. + +**Verification.** After `member_retired = 1`, `system.cas_mounts` shows the member's +row as `retired` with `retirement_verified = 1` (an unfinished run leaves `retirement_verified = 0`); a `SYSTEM CAS GC RUN` no longer waits on its heartbeat. Run it on the GC leader (`is_leader = 1` in `system.cas_mounts`); on another node it does nothing. See [mount, unmount, crash](/antalya/cas/architecture/mounts-and-leases#mount-lifecycle) for how the claim, drain, and retirement steps fit into the mount-slot lifecycle. + +### Bringing a member back {#rejoin} + +Restart the server. A server whose `server_root_id` was decommissioned rejoins the pool on its next start, +with the same or a new `server_uuid`. It comes back empty: its local table definitions attach with no data. + +The start rejoins only after the decommission finished: + +- the last run reported `member_retired = 1`, and `system.cas_mounts` shows the member as `retired` with + `retirement_verified = 1`; +- GC removed the catalog rows and manifests of the old life: run `SYSTEM CAS GC RUN ''` on the GC leader + (`is_leader = 1` in `system.cas_mounts`) after the verified run; it can take several rounds. + +| Start fails with | Action | +|---|---| +| `cannot rejoin` | `epoch` or `mount` is missing: the id was retired by 26.6.4 or the object was deleted by hand. Configure a new `cas_server_root_id` | +| `decommission is not finished` | finish the decommission under the [rerun rule](#rejoin-rerun-rule) until `member_retired = 1`, then restart | +| `previous life remains` | the message names the part left, and for catalog rows the first namespaces and their state. Catalog rows or manifests: run `SYSTEM CAS GC RUN ''` on the GC leader, `is_leader = 1` (it can take several rounds) and restart. A row that stays `removing` for many rounds is held by GC: the leader counts it in `CASGCStuckRemovals` and its stuck-removal warning names the reason. An object under `roots//`: delete it by hand and restart | +| `still running` | a decommission run, a killed run or an interrupted restart holds the slot: wait for the run to finish, or for the lease to lapse and GC to fence it (up to one GC round after the lease threshold), then restart | +| `kept changing` | other starts or a decommission run kept rewriting the owner or the slot during this start: restart | +| `no server uuid` | the process (for example `clickhouse disks` before a uuid file exists) has no server uuid and never rejoins: start the server that should own the id | + +Keep `/var/lib/clickhouse/uuid` (the server uuid file) on a persistent volume. A lost uuid file makes the +server start as a different server: a `server_root_id` that is not decommissioned refuses it as owned by a +different server, and a decommissioned one is rejoined under the new `server_uuid`. + +#### Rerun rule {#rejoin-rerun-rule} + +Rerun `SYSTEM CAS DROP POOL MEMBER` to finish a decommission only while both hold: + +- `system.cas_mounts` shows the id as `retired` with `retirement_verified = 0`; +- no decommission of that id is running. + +A run killed after its tombstone leaves the row `live`, then `expired`, then `retired` with +`retirement_verified = 0` once GC fences its lease, up to one GC round after the lease threshold. A run killed +before its tombstone ends `fenced` with the owner not retired. Do not rerun on any other state. `retired` with +`retirement_verified = 1` is finished: restart the member to rejoin. A `live`, `expired`, `terminated` or +`fenced` row belongs to whoever owns the id now, a rejoined member included, and a rerun would delete its data. +A run that warns `the tombstone was rolled back` lost its claim and left the id unretired: running the command +again is a new decommission of the id. A run that warns `the admin claim was lost during the drain` keeps its tombstone and +is finished by a rerun under this rule. + +A tombstone can land late: the store may apply it after the run reported an error. If a server of the id ran +after the tombstone time printed in its start error, do not rerun: the rerun deletes what that server wrote. +Instead keep the owner object's header line, replace its body line with +`{"server_uuid":"","owner_life":""}` (the same `server_uuid`, the life from the error plus one) +and restart that member. + +### Upgrading and downgrading {#decommission-compatibility} + +| Case | Behaviour | +|---|---| +| A `server_root_id` retired by 26.6.4 (its `epoch` and `mount` deleted) | not repaired: a decommission rerun refuses the absent counter and it cannot rejoin; use a new `server_root_id` | +| A 26.6.4 decommission that stopped midway | the same refusal; use a new `server_root_id` | +| Flat 26.6.4 staging keys `staging//.tmp` | swept as epoch 0 at the next mount | +| Downgrade to 26.6.4 with a retired `server_root_id` kept by this version | 26.6.4 refuses to start it and its GC treats the farewell as terminal | +| Downgrade: `owner_life`, `retirement_verified`, epoch staging keys | skipped by 26.6.4 readers; staging swept recursively | +| Mixed versions: a 26.6.4 decommission tool rewrites an owner | the life drops to 0; servers of this version fail their owner check and stop (fail-closed) | +| Downgrade: a `server_root_id` rejoined under a new `server_uuid` | start it once on this version before downgrading; 26.6.4 then starts it normally | +| Downgrade: a `server_root_id` rejoined under the same `server_uuid` | 26.6.4 starts it and continues the counter | +| Mixed versions: a 26.6.4 server starts with a retired `server_root_id` | refused, as in 26.6.4: it cannot rejoin | +| Mixed versions: a 26.6.4 member restarts during a decommission | its owner check refuses after the tombstone; before it there is no recheck, as in 26.6.4 | + +Rolling upgrades keep the old races on the old binaries; decommission from an upgraded server. + +## Recreating a member whose id contains `/` {#recreate-id-with-slash} + +A `cas_server_root_id` that contains `/` is refused: the disk fails to open with `BAD_ARGUMENTS` and a +message that contains `contains '/'`. Pools created by earlier builds can hold such an id. To bring the +server back: + +1. Set `` to a new value without `/`, for example `shard-01-replica-a` instead of + `shard-01/replica-a`. If the id comes from macros, remove the `/` from the template. +2. Restart the server. The disk claims the new id over an empty subtree. + +Data stored under the old id is not visible under the new id. The old id's objects stay in the pool +and keep their blobs referenced. `SYSTEM CAS DROP POOL MEMBER` and `clickhouse-disks cas-drop-member` +refuse an id with `/`, so they cannot reclaim them. diff --git a/docs/en/antalya/cas/operations/monitoring.md b/docs/en/antalya/cas/operations/monitoring.md index 4a104120697b5..0984719047ba3 100644 --- a/docs/en/antalya/cas/operations/monitoring.md +++ b/docs/en/antalya/cas/operations/monitoring.md @@ -58,7 +58,7 @@ window and correlate them with the `server_root_id` in `system.cas_log`. | `CASMountRenewalRetries` | One per physical renewal `PUT` after the first in the same logical renewal | Positive growth shows in-period retry, not a later cadence beat | | `CASMountRenewalResolved` | One per logical renewal proved committed by an exact resolving `GET` | A response was ambiguous, but exact bytes and `write_attempt_id` proved the write | | `CASMountRenewalRecovered` | One per logical renewal committed after a retry or exact resolving `GET` | Recovered object-store blips that retained the existing mount incarnation | -| `CASMountRenewalDeadlineExceeded` | One per logical renewal stopped by the external lease-safety deadline | The last confirmed lease no longer left enough safe time; this is narrower than request-budget exhaustion | +| `CASMountRenewalDeadlineExceeded` | One per bounded logical renewal (startup or remount re-anchor, or a decommission's background renewal) stopped by the external lease-safety deadline | Growth means a bounded renewal found too little confirmed lease left; a mount's own background renewal never counts here | | `CASRemountAttempts` | One per invocation of the existing whole-chain remount attempt | Includes both successful and failed attempts | | `CASRemountSucceeded` | One per whole-chain attempt that restored `Live` under a fresh writer epoch | Must be a subset of `CASRemountAttempts` | | `CASRemountFailed` | One per whole-chain attempt that returned without restoring `Live` | Includes a named step exception or a step that returned transiently | diff --git a/docs/en/antalya/cas/operations/troubleshooting.md b/docs/en/antalya/cas/operations/troubleshooting.md index 1113ec1855dd4..5ac781941c372 100644 --- a/docs/en/antalya/cas/operations/troubleshooting.md +++ b/docs/en/antalya/cas/operations/troubleshooting.md @@ -17,17 +17,62 @@ tools. | Symptom | Diagnosis | Action | |---|---|---| | A server keeps losing its mount lease and self-remounting | Check `system.cas_mounts` for the server's own `state`/`expires_at`, then correlate `watermark_renew` and `mount_remount` in `system.cas_log`; losing the lease trips a local fence and latches a remount generation | Read the failed renewal's `classification` before changing anything — it alone now says why (see [the decision flow](#mount-renewal-remount-flow)). Look for object-store latency consuming the confirmed lease or BOOTTIME advancement; see [the mount lease](/antalya/cas/architecture/mounts-and-leases#mount-lease) | -| Writes slow down or stall under load, with no exception reaching the client | S3 `SlowDown`/`ServiceUnavailable`/`RequestTimeout`/`InternalError` (5xx) responses are not on the request engine's `isDefinitelyRefusedWrite` definite-failure list (only malformed-request, entity-too-large, and access-denied that no credential refresh can fix are), so they classify as ambiguous and are retried automatically. Confirm with `sum(ProfileEvents['CASConditionalWriteUnresolved'])` rising alongside `sum(ProfileEvents['CASConditionalWriteAttempts'])` over `system.query_log` for the affected window (or `ProfileEvent_CASConditionalWriteUnresolved` in `system.metric_log` for a cumulative view across queries), and check `system.blob_storage_log` for `disk_name = ''` rows with a nonzero `error_code` around the same window | Nothing to configure per-request: the request engine retries the same `(key, bytes)` with capped-exponential backoff (200ms initial, capped at 5s, full jitter) until the 90-second operation deadline — there is no separate attempts ceiling, only the deadline — and the mount-lease renewer keeps extending the fence across the disruption — this is the "blips, throttling, partial outages" case the write path is built to survive. Confirm the mount lease itself is still renewing (`system.cas_mounts.expires_at` moving forward, `last_success_age_seconds` not climbing) — if it is, this is expected and self-resolving. If `SlowDown` responses are sustained rather than transient, check the bucket's request-rate limits against the pool's actual PUT/GET rate (see [bucket requirements](/antalya/cas/bucket-requirements)) and consider lowering `cas_blob_upload_pool_size` to reduce concurrent upload traffic; a write only surfaces a client-visible `NETWORK_ERROR` if the 90-second deadline is exhausted before the store recovers, and that error is retried by the ordinary merge/insert backoff, not silently dropped | -| `GC` never seems to reclaim space after tables are dropped | `SELECT * FROM system.cas_gc_log WHERE event_type='Finish' ORDER BY event_time DESC LIMIT 5` — check `outcome`; also `SELECT is_leader FROM system.cas_mounts` on this node | If `outcome != 'Success'`/`'Deferred'`, see [reading GC health](/antalya/cas/operations/monitoring#gc-health); if this node is not the leader (`is_leader = 0`), it never reclaims for this disk — check the peer holding leadership. Reclamation also needs at least two full rounds past condemnation by design (the grace period is rounds, not acks) — a single manual `SYSTEM CAS GC RUN` will not finish it | +| Writes slow down or stall under load, with no exception reaching the client | S3 `SlowDown`/`ServiceUnavailable`/`RequestTimeout`/`InternalError` (5xx) responses are not on the request engine's `isDefinitelyRefusedWrite` definite-failure list (only malformed-request, entity-too-large, and access-denied that no credential refresh can fix are), so they classify as ambiguous and are retried automatically. Confirm with `sum(ProfileEvents['CASConditionalWriteUnresolved'])` rising alongside `sum(ProfileEvents['CASConditionalWriteAttempts'])` over `system.query_log` for the affected window (or `ProfileEvent_CASConditionalWriteUnresolved` in `system.metric_log` for a cumulative view across queries), and check `system.blob_storage_log` for `disk_name = ''` rows with a nonzero `error_code` around the same window | Nothing to configure per-request: the request engine retries the same `(key, bytes)` with capped-exponential backoff (200ms initial, capped at 5s, full jitter) until the 90-second operation deadline — for these conditional writes there is no separate attempts ceiling, only the deadline. A blob body upload is different: it is never resent as is after an ambiguous outcome, and each retry publishes a fresh envelope, up to 8 attempts, and no new attempt starts after the 90-second deadline. An attempt already started can run longer: each creating request waits up to the disk's `request_timeout_ms`, and multipart part uploads keep their own retries. `CASBlobPublishAttemptFailed` counts failed attempts; `CASBlobPublishGaveUp` counts uploads that failed at least once and then ended with an exception, at the attempt cap or the deadline. Meanwhile the mount-lease renewer keeps extending the fence across the disruption — this is the "blips, throttling, partial outages" case the write path is built to survive. Confirm the mount lease itself is still renewing (`system.cas_mounts.expires_at` moving forward, `last_success_age_seconds` not climbing) — if it is, this is expected and self-resolving. If `SlowDown` responses are sustained rather than transient, check the bucket's request-rate limits against the pool's actual PUT/GET rate (see [bucket requirements](/antalya/cas/bucket-requirements)) and consider lowering `cas_blob_upload_pool_size` to reduce concurrent upload traffic; a write only surfaces a client-visible `NETWORK_ERROR` if the 90-second deadline passes before the store recovers (an attempt started before the deadline can finish after it), and that error is retried by the ordinary merge/insert backoff, not silently dropped | +| A write fails with a message containing `CAS ref-log txn`, `is durable but was not acknowledged`, `outcome is unknown` or `nothing is durable` | The phrase says whether the write can still take effect. A write blocked behind an earlier unresolved transaction (a wedged or needs-recovery lane) was never attempted, and its message names the earlier transaction's outcome with the same phrase. Errors that name no earlier transaction carry no phrase, for example an admission refusal before the write is queued (`the operation is no longer admitted`), a failed ref-table recovery, a lost mount lease, a table superseded by a self-remount, and a lane in a terminal state (`INVALID_STATE`, "terminal lane state") | Read [CAS commit errors](#cas-commit-errors) before retrying: two of the three outcomes can duplicate on retry | +| `GC` never seems to reclaim space after tables are dropped | `SELECT * FROM system.cas_gc_log WHERE event_type='Finish' ORDER BY event_time DESC LIMIT 5` — check `outcome`; also `SELECT is_leader FROM system.cas_mounts` on this node | If `outcome != 'Success'`/`'Deferred'`, see [reading GC health](/antalya/cas/operations/monitoring#gc-health); if this node is not the leader (`is_leader = 0`), it never reclaims for this disk — check the peer holding leadership. Reclamation also needs at least two full rounds past condemnation by design (the grace period is rounds, not acks) — a single manual `SYSTEM CAS GC RUN` will not finish it, and it does nothing on a node that is not the leader | | A dangling-access exception or `CORRUPTED_DATA` on read | Run `clickhouse-disks cas-fsck --detail` and check `dangling` specifically — it is the one class that means data loss, distinct from `unreachable`/`awaiting-gc`, which are just waiting for graduation | A nonzero `dangling` count is a real incident: collect the `--detail` output (see [what to collect before filing a bug](/antalya/cas/operations/debugging#filing-a-bug)) before taking any destructive action | | `SYSTEM CAS FSCK` or `clickhouse-disks cas-fsck` times out on a large pool | The scan is bounded by `--timeout` (default 600s / the `SYSTEM` form has no override); a large `roots/` prefix can make the scan slow | Retry with `--partial` to see the counts accumulated so far instead of aborting empty-handed, or `--namespace ` to scope the scan to a subset of namespaces | -| `SYSTEM CAS DROP POOL MEMBER` returns a non-empty `warnings` column | A per-object drain step could not confirm emptiness; the mount slot is left terminated but not fully drained, as a resume anchor | Rerun the same command — it is resumable and skips namespaces already marked removed, reporting them under `namespaces_already_removed` | +| `SYSTEM CAS DROP POOL MEMBER` returns `member_retired = 0` or a non-empty `warnings` column | The run stopped before its verified mark: a drain step failed, the admin lost its claim, or the farewell could not be confirmed; the owner keeps an unverified tombstone as the resume anchor. Only a claim lost before the drain started rolls the tombstone back, with the warning `the tombstone was rolled back`; nothing was removed then. After `the admin claim was lost during the drain` part of the member's data may be gone: the tombstone stays and the member does not start until a rerun verifies the retirement. A warning `the owner moved on to life N` means the owner was un-retired after this run's tombstone, by a rejoin or by another run's rollback | Rerun the same command only under the [rerun rule](/antalya/cas/operations/migration#rejoin-rerun-rule); it resumes over its own tombstone. A rerun refused as alive or contended waits until GC fences the previous run's slot: about one mount lease TTL (`cas_mount_lease_ttl_ms`) plus one GC round, with a GC leader running. `namespaces_pending_gc` alone is not a reason to rerun | +| `SYSTEM CAS DROP POOL MEMBER` refuses with `alive or contended` although the member is down | The claim trusts no wall clock: `expired` in `system.cas_mounts` is not a certificate of death; only a terminal slot (`terminated` or `fenced`) is reclaimable | Wait until the member's row shows `state = 'terminated'` (clean stop) or `state = 'fenced'` (the disk's GC leader fences the stale lease at the start of a GC round, once it has seen it unchanged for about one `cas_mount_lease_ttl_ms` plus one `cas_mount_renew_period_ms`, so the wait can be one GC round longer; `SYSTEM CAS GC RUN` on the leader, `is_leader = 1` in `system.cas_mounts`, can trigger the fence, but the run that fences must come at least that long after the first run that saw the lease), then rerun | +| A server refuses to start with `owned by a different server` | The owner object binds the `server_root_id` to another `server_uuid`: the local uuid file was regenerated, or two servers share one `cas_server_root_id` | Restore the uuid file (keep it on a persistent volume); or start this server under a new `cas_server_root_id`. To reclaim the old id's data once its owning server is permanently gone, decommission it with `SYSTEM CAS DROP POOL MEMBER` from a surviving member, once its slot is `terminated` or `fenced`; the two steps are independent, and in a single-member pool starting under the new id comes first, because a hard-killed old id is fenced only by a running GC leader | +| A `CAS` disk refuses to open with `contains '/'` | Its `cas_server_root_id` contains `/`, directly or through a macro; such ids are refused, including ids that already own their owner object in the pool | Configure a new `cas_server_root_id` without `/` and restart; see [recreating a member whose id contains `/`](/antalya/cas/operations/migration#recreate-id-with-slash) | +| A server fails to start with `decommission is not finished`, `previous life remains`, `still running`, `kept changing`, `no server uuid` or `cannot rejoin` | Its `server_root_id` was decommissioned and the rejoin cannot proceed: the retirement is unverified or not drained, a run still holds the slot, concurrent writers kept changing the owner or slot, the process has no server uuid, or the id cannot be rejoined | See [bringing a member back](/antalya/cas/operations/migration#rejoin) | +| `system.cas_mounts.lifecycle = 'identity_lost'` after a remount, with `the owner moved from life N to life M` in the log | A rejoin or a decommission rollback rewrote the owner while this process was fenced; it cannot resume under a life it did not start under | Restart the server; it starts under the current life | +| `system.cas_mounts.lifecycle = 'identity_lost'` after a remount, with `was decommissioned while this process was fenced` in the log | `SYSTEM CAS DROP POOL MEMBER` retired this `server_root_id` while the process was paused or cut off from the store. The process writes nothing more | Stop the server. A restart refuses with `was decommissioned`; to bring the member back, see that row | | Writes or `ALTER`s on a `CAS` disk fail with a `READONLY`-class error | The disk's metadata storage rejects every mutating entry point; this is deliberate for a disk opened with `true`, used by every offline `clickhouse-disks` tool | Confirm whether the disk was intentionally configured read-only (offline inspection, `cas-fsck`, `cas-gc-dryrun`, `cas-gc-rebuild`, `cas-drop-member` all require it); a production disk serving writes must not carry `true` | | A table stays unavailable after a transient network error during startup | `AsyncLoader` has no retry/requeue path for a failed table load job: a transient S3 `NETWORK_ERROR` during `CAS` ref-table startup recovery can leave the job permanently `FAILED` | Restart the server, or issue a fresh load for the table; this is a one-shot job design, not a `CAS`-specific bug | -| A mounted pool directory was removed or renamed out of band | Renewal observes an absent, foreign, successor, or otherwise conflicting mount body and terminates the keeper with a typed fail-closed exception; the runtime closes the local write fence and requests remount rather than adopting the body | Never remove or rename a live pool's storage path. To retire a member permanently use [`SYSTEM CAS DROP POOL MEMBER`](/antalya/cas/operations/migration#decommission) instead of raw filesystem operations; collect the `watermark_renew` classification (`vanished` for an absent body, `conflict` for a foreign, successor, or otherwise conflicting one) and the subsequent `mount_remount` step | +| A mounted pool directory was removed or renamed out of band | Renewal observes an absent, foreign, successor, or otherwise conflicting mount body and terminates the keeper with a typed fail-closed exception; the runtime closes the local write fence and requests remount rather than adopting the body | Never remove or rename a live pool's storage path. To remove a member, use [`SYSTEM CAS DROP POOL MEMBER`](/antalya/cas/operations/migration#decommission) instead of raw filesystem operations; collect the `watermark_renew` classification (`vanished` for an absent body, `conflict` for a foreign, successor, or otherwise conflicting one) and the subsequent `mount_remount` step | | Stale-looking part metadata after an out-of-band change to the pool | The part-folder view cache may be serving a retained (not re-validated) view | Set the disk-level `cas_part_folder_cache_bytes = 0` to disable view retention and `cas_manifest_decode_cache_bytes = 0` to make every manifest read fetch the body, then run `fsck`; both are diagnostic kill switches, not steady-state settings | | A wide merge (many thousands of columns) fails with a port-exhaustion error from the network layer | Each column in a wide part can cost a separate object-store operation in one merge, and a very wide part can issue on the order of the column count in requests, exhausting local ephemeral TCP ports under load | Reduce concurrent merge parallelism on that table, or increase the host's ephemeral port range; this is a general high-fan-out-merge limit, not specific to content addressing | +## CAS commit errors and retries {#cas-commit-errors} + +A failed ref-log commit of this write names one of three outcomes in its message. The error code, usually +`NETWORK_ERROR` (210), does not tell them apart; the phrase does. The original error is kept inside +the message, and the server log may also show the inner `CAS write could not be committed` line. +Each phrase describes the ref-log transaction that carried this part's change; one transaction can +batch several callers. A write blocked behind an earlier unresolved transaction (a wedged or +needs-recovery lane) was never attempted. Its message names the earlier transaction's outcome with the +same phrase; the phrase then describes that earlier transaction. Errors that name no earlier transaction (admission refusal before queueing, failed table recovery, +lost lease, self-remount supersession, terminal-lane `INVALID_STATE`) carry no phrase. + +| Message contains | What happened to this transaction | Retrying ("it" is the write the phrase describes) | +|---|---|---| +| `nothing is durable, so a retry cannot duplicate this transaction` | The store refused the write, or it was refused before any request was sent | Cannot duplicate this transaction. See the deduplication note below | +| `outcome is unknown` | A request may have been sent and nothing settled its result. The append lane stays wedged until the same key resolves, unless the resolution finds a foreign object, which faults the lane and schedules a remount | May duplicate: this write may still take effect. For a blocked write, "it" is the earlier write, which may be this client's own earlier attempt; the blocked write itself was not attempted | +| `is durable but was not acknowledged` | It landed (named as `txn -`), then a later step failed: the `_ckpt` frontier publish, the mount fence, or the install admission. The next ref-log recovery installs it | May duplicate: the write may still take effect. What a promoted part becomes at a restart before part cleanup drops the ref depends on the engine; see below | +| `is durable but was not acknowledged` together with `the part is never published` | The same failure, at the precommit step of an `INSERT`. Recovery installs the precommit binding, which is later removed; the part never appears | The rows do not appear. Within `non_replicated_deduplication_window` a retry of the same block is skipped as a duplicate (see the deduplication note below), so the rows are lost | + +Scope: an `INSERT` that spans several partitions commits one part at a time. Earlier parts stay +committed; a failed statement does not roll them back. None of these phrases proves that re-running +the whole query is duplicate-free. + +For an `INSERT` in the last two rows, the part is rolled back in memory, so a `SELECT` right after +the error does not show its rows. If a promotion landed and the server restarts before part +cleanup drops the ref, the engine decides what the part becomes. `MergeTree` loads it as active, +and its rows appear. `ReplicatedMergeTree` makes the part durable before registering it in Keeper +(`MergeTreeData::Transaction::renameParts` runs before the Keeper multi), so the part has no +Keeper entry: at startup it is an unexpected part and is moved to `detached/ignored_`. Its +rows do not appear, and because the block id never reached Keeper a retry is not deduplicated; it +lands a second copy only if the detached part is later attached by hand. A failure at the +precommit step never publishes the part, and its message says so. + +`non_replicated_deduplication_window` does not make these retries safe. `MergeTree` writes the +block id to the deduplication log before the CAS commit and does not remove it when the commit +fails. While the matching block ID remains in the deduplication log, a retry of the same block is +reported as success and skipped as a duplicate, so its rows appear only if the original write +takes effect. + ## Mount renewal and remount decision flow {#mount-renewal-remount-flow} Start with the `watermark_renew` timeline described in @@ -38,16 +83,21 @@ Start with the `watermark_renew` timeline described in `CASMountRenewalRecovered` rises while `CASMountLeaseLost` and all remount counters stay flat. No intervention is needed unless the rate is sustained; investigate backend throttling/latency before the blips consume the lease budget. -2. **External lease-safety exhaustion.** The failed row has - `classification = 'external_lease_deadline'`; `CASMountRenewalDeadlineExceeded` and - `CASMountLeaseLost` rise. The runtime correctly refused to manufacture authority beyond the last - confirmed lease. Check object-store latency and BOOTTIME/suspend history, then follow the ensuing - remount. `classification = 'request_deadline'` is the sibling case: the ninety-second request - policy exhausted first rather than the lease's own safety margin. -3. **Cancellation.** `classification = 'cancelled'` after a sent request is terminal and suppresses a - clean farewell because the request may still land. Cancellation before any request remains - `Active` and emits no failed aggregate row; during graceful shutdown that is the expected - clean-release path. +2. **External lease-safety exhaustion.** Only a bounded renewal ends this way: the startup re-anchor, + the remount re-anchor, or the background renewal of a pool opened for decommission. A mount's own + background renewal keeps retrying; a long outage shows as + `lifecycle_reason = 'lease_expired'` and resumes under the same `writer_epoch`. The failed row has + `classification = 'external_lease_deadline'` and `CASMountRenewalDeadlineExceeded` rises. The + runtime refused to claim authority beyond the last confirmed lease. Check object-store latency and + BOOTTIME/suspend history. At startup the disk does not start; during a remount the remount loop + retries. `classification = 'request_deadline'` is the sibling case for the same bounded renewals: + the ninety-second request policy exhausted first rather than the lease's own safety margin. +3. **Cancellation.** `classification = 'cancelled'` after a request of the current engine call was sent + is terminal and suppresses a clean farewell because the request may still land; so is cancellation + while the reads after a refused precondition repeat. Cancellation before any request, or between two + calls of the background renewal after a refusal, remains `Active` and emits no failed aggregate row; + during graceful shutdown that is the expected clean-release path while the lease has room for the + farewell. During a pause it has none: the slot is left for observation. 4. **Confirmed conflict.** `classification = 'conflict'` means exact resolution found another body; inspect `server_root_id`, `writer_epoch`, `seq`, and `write_attempt_id`. Same-pair twins, GC-fenced bodies, successor epochs, and foreign holders all remain fail closed. Do not delete or rewrite the diff --git a/docs/en/antalya/metrics_and_events.md b/docs/en/antalya/metrics_and_events.md index 7e968baf16c66..5985022bb7d5a 100644 --- a/docs/en/antalya/metrics_and_events.md +++ b/docs/en/antalya/metrics_and_events.md @@ -63,7 +63,7 @@ Pool lifecycle and teardown. A non-zero value here is an incident signal. | `CASDataRootVanished` | `Number` | Counts CAS pools that entered a terminal Vanished lifecycle state (replaced / forgotten): the pool's data root was proven foreign (pool_id mismatch) or was decommissioned by SYSTEM CAS FORGET. The disk stays registered but store-class access fails loud with a typed error naming the sub-state; restart re-registers the name. | | `CASDetachedWorkDrainTimeouts` | `Number` | Counts CAS storage teardowns whose bounded wait for detached background work expired with work still in flight. The teardown proceeds, but for that teardown it could not be established that no tracked task still holds the pool. Expected to stay at zero. | | `CASEventDroppedContextExpired` | `Number` | Number of CAS system-log events dropped because the storage's `Context` reference expired before delivery. A non-zero value indicates event production outlived the owning server context. | -| `CASIdentityLost` | `Number` | Counts CAS pools that entered the IdentityLost lifecycle state: the pool sentinels (_pool_meta and the owner anchor) were observed authoritatively absent (both KeyAbsent). IdentityLost is a fail-loud TERMINAL state -- store-class access fails loud from this point and this pool's background threads (remount + GC) self-exit; there is no observer. Recover by restart or SYSTEM CAS FORGET; a matching-sentinel restore does not auto-revive. | +| `CASIdentityLost` | `Number` | Counts CAS pools that entered the IdentityLost lifecycle state: the pool sentinels (_pool_meta and the owner anchor) were observed authoritatively absent (both KeyAbsent), or a self-remount found the owner at another life. IdentityLost is a fail-loud TERMINAL state -- store-class access fails loud from this point and this pool's background threads (remount + GC) self-exit; there is no observer. Recover by restart or SYSTEM CAS FORGET; a matching-sentinel restore does not auto-revive. | #### Blob requests {#cas-blob-requests} @@ -83,6 +83,8 @@ validation, and other callers. | `CASBlobHeadMiss` | `Number` | Number of successful backend HEAD requests under CAS blob paths that found no object, aggregated across all callers. | | `CASBlobList` | `Number` | Number of CAS blob LIST requests. Growing values indicate more object enumeration. | | `CASBlobOverwrite` | `Number` | Number of CAS blob overwrite requests. Growing values indicate repeated replacement writes. | +| `CASBlobPublishAttemptFailed` | `Number` | Number of CAS blob publication attempts that failed with an ambiguous or retryable error. The writer observes the blob again and publishes a fresh envelope, up to the attempt limit. | +| `CASBlobPublishGaveUp` | `Number` | Number of CAS blob publications that ended with an exception after at least one failed attempt, for example at the attempt limit or the deadline. | | `CASBlobPut` | `Number` | Number of CAS blob PUT requests. Grows with blob uploads and indicates storage write traffic. | | `CASBlobPutDeduplicated` | `Number` | Number of CAS blob deduplicating PUT requests. Grows when uploads reuse existing content. | @@ -270,9 +272,9 @@ These counters cover per-blob metadata creates, rewrites, and deletes. | Event | Type | Description | | --- | --- | --- | -| `CASMetaAdoptBackfill` | `Number` | Number of blob-adoption paths that attempted a missing-metadata Clean backfill. Counts the attempt, not a guaranteed metadata creation. | +| `CASMetaAbsentRepublish` | `Number` | Number of blob publication attempts started because the body was present but its freshness marker was absent; such a body is republished, never adopted. Counts attempts, including ones that fail. | | `CASMetaCompareSwap` | `Number` | Number of conditional CAS blob-metadata rewrites, including lost races. Growing values indicate metadata updates or contention. | -| `CASMetaCreateClean` | `Number` | Number of absent-body publication paths that entered Clean metadata reconciliation. Counts the reason entry, not a guaranteed metadata creation. | +| `CASMetaCreateClean` | `Number` | Number of publications after an absent body or an absent marker that entered Clean metadata reconciliation. Counts the reason entry, not a guaranteed metadata creation. | | `CASMetaDelete` | `Number` | Number of exact-token CAS blob-metadata delete attempts. Growth indicates GC or cleanup activity. | | `CASMetaPut` | `Number` | Number of conditional CAS blob-metadata create attempts, including lost races. Growth indicates metadata reconciliation or write contention. | | `CASMetaResurrectClean` | `Number` | Number of condemned-body replacement paths that entered Clean metadata reconciliation. Counts the reason entry, not a guaranteed metadata reset. | @@ -285,10 +287,10 @@ window. | Event | Type | Description | | --- | --- | --- | | `CASMountExclusivityViolation` | `Number` | Counts CAS mount releases where a runtime that still BELIEVED IT OWNED the mount (no deposition ever observed) found a FOREIGN occupant in the slot. This is the single-writer guarantee being broken, not a failover: the release refuses, the slot is left untouched, and the write fence is latched so the runtime stops trusting itself. This must always be zero. | -| `CASMountLeaseLost` | `Number` | Counts exactly once per operational CAS mount-lease Live-to-TransientNotLive loss/recovery generation. The initiating external loss or the first ordinary terminal renewal consumer owns the increment, including external lease-safety deadline exhaustion; parked/classification/shutdown paths do not duplicate it. | +| `CASMountLeaseLost` | `Number` | Counts exactly once per operational CAS mount-lease Live-to-TransientNotLive loss/recovery generation. The initiating external loss or the first ordinary terminal renewal consumer owns the increment; parked/classification/shutdown paths do not duplicate it. | | `CASMountReleaseSkippedForeignOccupant` | `Number` | Counts conclusive CAS mount-renewal observations that found a FOREIGN successor in the slot. This is the EXPECTED end state of a failover: renewal fences the deposed runtime, terminal teardown skips the farewell without backend I/O, and the successor's slot is left byte-for-byte untouched. Steady non-zero values on a cluster that is not failing over are worth investigating; a value that tracks failovers is normal. | | `CASMountRenewalAttempts` | `Number` | Number of physical conditional renewal PUTs sent for CAS mount leases. This counts transport attempts, not logical renewals. | -| `CASMountRenewalDeadlineExceeded` | `Number` | Number of logical CAS mount-lease renewals stopped by the external lease-safety deadline. Growth means the last confirmed lease no longer had enough safe time for another physical attempt. | +| `CASMountRenewalDeadlineExceeded` | `Number` | Number of bounded CAS mount-lease renewals (the startup and remount re-anchors, and the background renewal of a pool opened for decommission) stopped by the external lease-safety deadline. Growth means the last confirmed lease no longer had enough safe time for another physical attempt. | | `CASMountRenewalRecovered` | `Number` | Number of logical CAS mount-lease renewals that committed after a physical retry or exact resolving GET. | | `CASMountRenewalResolved` | `Number` | Number of CAS mount-lease renewals whose committed outcome was proved by an exact resolving GET. | | `CASMountRenewalRetries` | `Number` | Number of physical conditional renewal PUTs sent after the first attempt of one logical CAS mount-lease renewal. | @@ -524,7 +526,7 @@ These counters cover client-side time of individual S3 and Azure list and head c | `AzureListObjectsMicroseconds` | `Microseconds` | Time of Azure blob storage API ListObjects execution. | | `S3HeadObjectMicroseconds` | `Microseconds` | Time of S3 API HeadObject execution. | | `S3ListObjectsMicroseconds` | `Microseconds` | Time of S3 API ListObjects execution. | -| `S3SingleAttemptRetryConsultations` | `Number` | Number of AWS SDK retry consultations refused by the single-attempt S3 retry profile. Non-zero means the SDK attempted a transparent retry on a write that must make exactly one HTTP attempt. | +| `S3SingleAttemptRetryConsultations` | `Number` | Number of AWS SDK retry consultations refused by a single-attempt S3 client: the clone used by the single-attempt retry profile, or the clone that sends the object-creating requests of a write that forbids a resend. Non-zero means a request on such a client failed and was not resent. | ### Other profile events {#other-profile-events} diff --git a/docs/en/operations/system-tables/cas_mounts.md b/docs/en/operations/system-tables/cas_mounts.md index 5a9465db04af6..0aa8aad3ce473 100644 --- a/docs/en/operations/system-tables/cas_mounts.md +++ b/docs/en/operations/system-tables/cas_mounts.md @@ -32,15 +32,19 @@ rows. - `expires_at` ([DateTime64(3)](/sql-reference/data-types/datetime64)) — Time when the lease expires. - `min_active_build_sequence` ([UInt64](/sql-reference/data-types/int-uint)) — Oldest in-flight build sequence (`UINT64_MAX` means the mount said farewell). - `gc_fenced` ([UInt8](/sql-reference/data-types/int-uint)) — `1` if GC fenced this slot out (terminal). -- `state` ([String](/sql-reference/data-types/string)) — One of `live`, `expired`, `terminated`, `fenced`, `corrupt`. +- `state` ([String](/sql-reference/data-types/string)) — One of `live`, `expired`, `terminated`, `fenced`, `retired`, `corrupt`. `retired` is a terminated or fenced slot of a decommissioned member; `retirement_verified` says whether the decommission finished. `corrupt` covers an undecodable mount body, and a terminated or fenced slot whose owner object cannot be decoded. Each terminated or fenced slot costs one more `GET` for its owner; a `corrupt` row from an undecodable mount body costs none. - `is_leader` ([Nullable(UInt8)](/sql-reference/data-types/nullable)) — `1` if this server's GC scheduler currently holds this disk's leadership lease. - `pending_reclaim` ([Nullable(Int64)](/sql-reference/data-types/nullable)) — Cumulative two-phase deletion backlog observed by this process's GC on this disk (condemned entries minus executed exact-token deletes). - `last_success_age_seconds` ([Nullable(UInt64)](/sql-reference/data-types/nullable)) — Seconds since this disk's GC last led a round (`0` if it has never led or GC is not running here). - `wedged_namespace_count` ([Nullable(UInt64)](/sql-reference/data-types/nullable)) — Ref-append lanes currently wedged on this disk (an uncertain `PUT` exhausted its retry budget). - `lifecycle` ([String](/sql-reference/data-types/string)) — This server's content-addressed pool lifecycle for the disk (a non-gated snapshot, always populated so a not-live disk stays visible): one of `live`, `not_live`, `identity_lost`, `vanished`, `constructing` (never started), or `shutdown` (torn down). -- `lifecycle_reason` ([String](/sql-reference/data-types/string)) — The enum-clean sub-state word for a `vanished` disk: `replaced` or `forgotten`. Empty for every other lifecycle, so `lifecycle || '(' || lifecycle_reason || ')'` reads e.g. `vanished(forgotten)`. -- `lifecycle_detail` ([String](/sql-reference/data-types/string)) — The full typed reason text naming the actual cause when not live: the vanish diagnosis (a data root replaced by a foreign pool, or decommissioned by `SYSTEM CAS FORGET` at a given time) or the identity-loss message. Empty when live. -- `lifecycle_since` ([Nullable(DateTime)](/sql-reference/data-types/nullable)) — When this server entered the current non-live lifecycle state. `NULL` when live, or when the state has no backing pool to date from. +- `lifecycle_reason` ([String](/sql-reference/data-types/string)) — The enum-clean sub-state word: `replaced` or `forgotten` for a `vanished` disk, `lease_expired` for a `not_live` disk whose mount lease has too little time left to admit a write while its renewal keeps retrying. Empty for every other lifecycle, so `lifecycle || '(' || lifecycle_reason || ')'` reads e.g. `vanished(forgotten)`. +- `lifecycle_detail` ([String](/sql-reference/data-types/string)) — The full typed reason text naming the actual cause when not live: the vanish diagnosis (a data root replaced by a foreign pool, or decommissioned by `SYSTEM CAS FORGET` at a given time) or the identity-loss message. Empty when live, and for `not_live` with `lifecycle_reason = 'lease_expired'`. +- `lifecycle_since` ([Nullable(DateTime)](/sql-reference/data-types/nullable)) — When this server entered the current non-live lifecycle state. For `lease_expired`, when writes paused. `NULL` when live, or when the state has no backing pool to date from. +- `renewal_in_progress` ([Nullable(UInt8)](/sql-reference/data-types/nullable)) — `1` while this server renews its mount lease. +- `renewal_attempts` ([Nullable(UInt64)](/sql-reference/data-types/nullable)) — Conditional `PUT`s sent by the current renewal, or by the last one once it ended. Updated per attempt, so a renewal stuck in retries is visible before it ends. +- `last_renewal_error` ([Nullable(String)](/sql-reference/data-types/nullable)) — The last failed request of that renewal, as ` : `. `class` is `refused` when the store definitely refused the request (access denied, malformed request, entity too large), `transport` otherwise; `code` is the store's error name, such as `SlowDown`. `NULL` when no request failed. +- `retirement_verified` ([UInt8](/sql-reference/data-types/int-uint)) — `1` on a `retired` row whose decommission verified the retirement, `0` otherwise. A server started under that `server_root_id` rejoins only from `retired` with `retirement_verified = 1`. Rerun `SYSTEM CAS DROP POOL MEMBER` only on `retired` with `retirement_verified = 0`; see the [rerun rule](/antalya/cas/operations/migration#rejoin-rerun-rule). `lifecycle`/`lifecycle_reason`/`lifecycle_detail`/`lifecycle_since` are the SQL surface for diagnosing an identity-lost or forgotten disk without reading server logs — see the @@ -49,9 +53,10 @@ diagnosing an identity-lost or forgotten disk without reading server logs — se what each lifecycle value means, and [`SYSTEM CAS FORGET`](/antalya/cas/operations/debugging#sql-forget) for the command that produces `vanished(forgotten)`. -:::note Local-only GC-health columns -`is_leader`, `pending_reclaim`, `last_success_age_seconds`, and `wedged_namespace_count` are process-local -facts about *this* server's own GC scheduler. They are populated **only** on the row whose `server_root_id` matches +:::note Local-only columns +`is_leader`, `pending_reclaim`, `last_success_age_seconds`, `wedged_namespace_count`, +`renewal_in_progress`, `renewal_attempts` and `last_renewal_error` are process-local facts about +*this* server. They are populated **only** on the row whose `server_root_id` matches this server's own mount, and are `NULL` on every row describing another server's mount — stamping a local health fact onto a peer's row would misread as "the peer is the GC leader" during an incident. To see the peer's own view of these columns, query `system.cas_mounts` on that server. @@ -78,4 +83,4 @@ FORMAT Vertical; - [`system.cas_gc_log`](/operations/system-tables/cas_gc_log) — per-round GC event log. - [`system.cas_log`](/operations/system-tables/cas_log) — per-decision event log for the CAS garbage collector and writer. - [`SYSTEM CAS GC RUN`](/sql-reference/statements/system#system-cas-gc-run) — run one GC round synchronously. -- [`SYSTEM CAS DROP POOL MEMBER`](/sql-reference/statements/system#system-cas-drop-pool-member) — permanently decommission a dead pool member's `server_root_id`. +- [`SYSTEM CAS DROP POOL MEMBER`](/sql-reference/statements/system#system-cas-drop-pool-member) — decommission a dead pool member's `server_root_id`. diff --git a/docs/reference/statements/system.mdx b/docs/reference/statements/system.mdx index 2885ffd7bde77..24cd5a61ae96f 100644 --- a/docs/reference/statements/system.mdx +++ b/docs/reference/statements/system.mdx @@ -688,19 +688,24 @@ SYSTEM CAS FORGET [ON CLUSTER cluster_name] disk_name `disk_name` is **required**. It is an assertion, not a proof of erasure: the disk stays registered and answers further store-class access with a typed error, and a server restart re-registers the name. Returns no result set. This is different from -[`SYSTEM CAS DROP POOL MEMBER`](#system-cas-drop-pool-member), which permanently retires one pool -*member's* identity across the whole shared pool — `FORGET` only affects this node's own local view +[`SYSTEM CAS DROP POOL MEMBER`](#system-cas-drop-pool-member), which decommissions one pool +*member* across the whole shared pool — `FORGET` only affects this node's own local view of one disk. ### SYSTEM CAS DROP POOL MEMBER {#system-cas-drop-pool-member} -Permanently decommissions a dead member (`server_root_id`) of a content-addressed disk -pool. It claims the member's mount slot as an administrative writer — fencing that `server_root_id` from ever -writing again — then drops every table namespace the member owned, drains eligible manifest debris, -staging objects, and mountpoint objects belonging to it, and retires the mount slot once all drains -are confirmed. This is a **destructive, irreversible** operation: only run it once the `server_root_id` is -confirmed permanently dead, since it fences the member out even if it later comes back online, and -it deletes namespace and drain state that cannot be recovered. +Decommissions a dead member (`server_root_id`) of a content-addressed disk +pool. It claims the member's mount slot as an administrative writer — fencing the member's current life — tombstones the member's owner, drops every table namespace the member owned, drains +eligible manifest debris, staging objects, and mountpoint objects belonging to it, writes the member's +farewell and marks the retirement verified. The control objects (`owner`, `epoch`, `mount`) stay. This is a **destructive** operation: run it only once the member is confirmed dead, since the +namespace and drain state it deletes cannot be recovered. + +The claim accepts only a terminal mount slot (`state` `terminated` or `fenced` in +`system.cas_mounts`). A hard-killed member shows `expired` until the disk's GC leader fences it at the start of a GC round, once it has +seen the lease unchanged for about one lease TTL plus one renew period; the wait can be one GC round longer. Until then the command +refuses with `alive or contended`. `SYSTEM CAS GC RUN` on the leader (`is_leader = 1` in +`system.cas_mounts`) can trigger the fence, but the run that fences must come at least that long after +the first run that saw the lease; two back-to-back runs do not fence. It is a writer operation, not GC: it emits ordinary ref-edge deltas rather than inventing GC transitions, and it does not synchronously reclaim shared blob content — removing the ref edges only @@ -710,18 +715,25 @@ makes the now-unreferenced blobs eligible for an ordinary GC round to reclaim la SYSTEM CAS DROP POOL MEMBER 'server_root_id' FROM DISK 'disk_name' [ON CLUSTER cluster_name] ``` -Both `server_root_id` and `disk_name` are required string literals (a `server_root_id` is an opaque server-root path -that may contain `/`, not a plain identifier, so it cannot be written unquoted). +Both `server_root_id` and `disk_name` are required string literals (a `server_root_id` is an opaque string, not a +plain identifier, so it cannot be written unquoted). A `server_root_id` that contains `/` is refused with +`BAD_ARGUMENTS`. The operation is resumable: a rerun skips namespaces already marked `Removed` and reports them separately from namespaces newly removed by this invocation. Per-object drain failures are recorded -as warnings and leave the slot in a terminated-but-not-fully-drained state that a later invocation -can resume from, rather than raising an exception. +as warnings; the owner keeps an unverified tombstone that a later invocation resumes over, rather than +raising an exception. The command returns one row with columns `server_root_id`, `namespaces_removed`, `namespaces_already_removed`, `committed_refs_removed`, `precommits_removed`, `manifest_debris_removed`, `staging_objects_removed`, -`mountpoint_objects_removed`, `slot_removed`, and `warnings`. A non-empty `warnings` means some -drain was not confirmed and the mount slot was left in place as a resume anchor. +`mountpoint_objects_removed`, `member_retired`, `warnings`, and `namespaces_pending_gc`. +`member_retired = 1` means the member's retirement is verified. `member_retired = 0` means the run stopped +before its verified mark: rerun it only under the +[rerun rule](/antalya/cas/operations/migration#rejoin-rerun-rule). Rows under `namespaces_pending_gc` +are deleted by GC and need no rerun. + +A server started later under the same `server_root_id` rejoins as an empty member once the retirement is +verified and drained ([details](/antalya/cas/operations/migration#rejoin)). ## Managing ReplicatedMergeTree Tables {#managing-replicatedmergetree-tables} diff --git a/programs/disks/CommandCaDropMember.cpp b/programs/disks/CommandCaDropMember.cpp index 60b2472f5dcab..e03d8c13df0ed 100644 --- a/programs/disks/CommandCaDropMember.cpp +++ b/programs/disks/CommandCaDropMember.cpp @@ -57,12 +57,13 @@ class CommandCaDropMember final : public ICommand std::cout << "server_root_id=" << report.srid << "\n" << "namespaces_removed=" << report.namespaces_removed << "\n" << "namespaces_already_removed=" << report.namespaces_already_removed << "\n" + << "namespaces_pending_gc=" << report.namespaces_pending_gc << "\n" << "committed_refs_removed=" << report.committed_refs_removed << "\n" << "precommits_removed=" << report.precommits_removed << "\n" << "manifest_debris_removed=" << report.manifest_debris_removed << "\n" << "staging_objects_removed=" << report.staging_objects_removed << "\n" << "mountpoint_objects_removed=" << report.mountpoint_objects_removed << "\n" - << "slot_removed=" << (report.slot_removed ? "true" : "false") << "\n"; + << "member_retired=" << (report.member_retired ? "true" : "false") << "\n"; for (const auto & w : report.warnings) std::cout << "warning=" << w << "\n"; } diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 21a47a3f7b1c8..d4f57590fcada 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -829,6 +829,8 @@ The server successfully detected this situation and will download merged part fr M(CASBlobCompareSwapConflict,"Number of CAS blob compare-and-swap conflicts. A non-zero value indicates concurrent update contention.", ValueType::Number) \ M(CASBlobHead, "Number of successful backend HEAD requests under CAS blob paths that found an object, aggregated across all callers.", ValueType::Number) \ M(CASBlobHeadMiss, "Number of successful backend HEAD requests under CAS blob paths that found no object, aggregated across all callers.", ValueType::Number) \ + M(CASBlobPublishAttemptFailed, "Number of CAS blob publication attempts that failed with an ambiguous or retryable error. The writer observes the blob again and publishes a fresh envelope, up to the attempt limit.", ValueType::Number) \ + M(CASBlobPublishGaveUp, "Number of CAS blob publications that ended with an exception after at least one failed attempt, for example at the attempt limit or the deadline.", ValueType::Number) \ M(CASBlobBodyPutAvoided, "Number of physical CAS blob body publications avoided after a safe presence observation found the object present. Growing values indicate saved upload traffic.", ValueType::Number) \ M(CASBlobUploadFanoutBatches, "Number of CAS intra-part blob-upload fan-out invocations (one per part publish that had at least one unique blob to upload). Compare with CASBlobUploadFanoutTasks for the average number of unique blobs uploaded per part.", ValueType::Number) \ M(CASBlobUploadFanoutTasks, "Number of CAS blob-upload tasks dispatched to the fan-out pool (one per unique blob ref across all part publishes). Duplicate references within a part collapse to one task.", ValueType::Number) \ @@ -954,8 +956,8 @@ The server successfully detected this situation and will download merged part fr M(CASMetaPut, "Number of conditional CAS blob-metadata create attempts, including lost races. Growth indicates metadata reconciliation or write contention.", ValueType::Number) \ M(CASMetaCompareSwap, "Number of conditional CAS blob-metadata rewrites, including lost races. Growing values indicate metadata updates or contention.", ValueType::Number) \ M(CASMetaDelete, "Number of exact-token CAS blob-metadata delete attempts. Growth indicates GC or cleanup activity.", ValueType::Number) \ - M(CASMetaCreateClean, "Number of absent-body publication paths that entered Clean metadata reconciliation. Counts the reason entry, not a guaranteed metadata creation.", ValueType::Number) \ - M(CASMetaAdoptBackfill, "Number of blob-adoption paths that attempted a missing-metadata Clean backfill. Counts the attempt, not a guaranteed metadata creation.", ValueType::Number) \ + M(CASMetaCreateClean, "Number of publications after an absent body or an absent marker that entered Clean metadata reconciliation. Counts the reason entry, not a guaranteed metadata creation.", ValueType::Number) \ + M(CASMetaAbsentRepublish, "Number of blob publication attempts started because the body was present but its freshness marker was absent; such a body is republished, never adopted. Counts attempts, including ones that fail.", ValueType::Number) \ M(CASMetaResurrectClean, "Number of condemned-body replacement paths that entered Clean metadata reconciliation. Counts the reason entry, not a guaranteed metadata reset.", ValueType::Number) \ M(CASGCMetaOps, "Number of per-hash metadata operations executed by CAS GC. Growing values indicate more GC candidates or metadata work.", ValueType::Number) \ M(CASGCEnumerationPages, "Number of CAS GC LIST pages fetched while enumerating the object universe. Growing values indicate a larger universe or more frequent scans.", ValueType::Number) \ @@ -991,15 +993,15 @@ The server successfully detected this situation and will download merged part fr M(CASMountRenewalRetries, "Number of physical conditional renewal PUTs sent after the first attempt of one logical CAS mount-lease renewal.", ValueType::Number) \ M(CASMountRenewalResolved, "Number of CAS mount-lease renewals whose committed outcome was proved by an exact resolving GET.", ValueType::Number) \ M(CASMountRenewalRecovered, "Number of logical CAS mount-lease renewals that committed after a physical retry or exact resolving GET.", ValueType::Number) \ - M(CASMountRenewalDeadlineExceeded, "Number of logical CAS mount-lease renewals stopped by the external lease-safety deadline. Growth means the last confirmed lease no longer had enough safe time for another physical attempt.", ValueType::Number) \ - M(CASMountLeaseLost, "Counts exactly once per operational CAS mount-lease Live-to-TransientNotLive loss/recovery generation. The initiating external loss or the first ordinary terminal renewal consumer owns the increment, including external lease-safety deadline exhaustion; parked/classification/shutdown paths do not duplicate it.", ValueType::Number) \ + M(CASMountRenewalDeadlineExceeded, "Number of bounded CAS mount-lease renewals (the startup and remount re-anchors, and the background renewal of a pool opened for decommission) stopped by the external lease-safety deadline. Growth means the last confirmed lease no longer had enough safe time for another physical attempt.", ValueType::Number) \ + M(CASMountLeaseLost, "Counts exactly once per operational CAS mount-lease Live-to-TransientNotLive loss/recovery generation. The initiating external loss or the first ordinary terminal renewal consumer owns the increment; parked/classification/shutdown paths do not duplicate it.", ValueType::Number) \ M(CASRemountAttempts, "Number of invocations of the CAS whole-chain remount attempt.", ValueType::Number) \ M(CASRemountSucceeded, "Number of CAS whole-chain remount attempts that restored Live under a fresh writer epoch.", ValueType::Number) \ M(CASRemountFailed, "Number of CAS whole-chain remount attempts that returned without restoring Live, including caught step exceptions.", ValueType::Number) \ M(CASMountReleaseSkippedForeignOccupant, "Counts conclusive CAS mount-renewal observations that found a FOREIGN successor in the slot. This is the EXPECTED end state of a failover: renewal fences the deposed runtime, terminal teardown skips the farewell without backend I/O, and the successor's slot is left byte-for-byte untouched. Steady non-zero values on a cluster that is not failing over are worth investigating; a value that tracks failovers is normal.", ValueType::Number) \ M(CASMountExclusivityViolation, "Counts CAS mount releases where a runtime that still BELIEVED IT OWNED the mount (no deposition ever observed) found a FOREIGN occupant in the slot. This is the single-writer guarantee being broken, not a failover: the release refuses, the slot is left untouched, and the write fence is latched so the runtime stops trusting itself. This must always be zero.", ValueType::Number) \ M(CASRemountHeldTransient, "Counts CAS remount attempts that could not decide the pool's identity and were held transient: the `_pool_meta` probe was inconclusive or its body could not be decoded (a partially written object, an unreadable store, or a pool whose format this build no longer reads). The mount stays fenced closed and the remount loop retries; a value that keeps growing means the pool will never remount without operator action -- read the accompanying warning for the probe's own reason.", ValueType::Number) \ - M(CASIdentityLost, "Counts CAS pools that entered the IdentityLost lifecycle state: the pool sentinels (_pool_meta and the owner anchor) were observed authoritatively absent (both KeyAbsent). IdentityLost is a fail-loud TERMINAL state -- store-class access fails loud from this point and this pool's background threads (remount + GC) self-exit; there is no observer. Recover by restart or SYSTEM CAS FORGET; a matching-sentinel restore does not auto-revive.", ValueType::Number) \ + M(CASIdentityLost, "Counts CAS pools that entered the IdentityLost lifecycle state: the pool sentinels (_pool_meta and the owner anchor) were observed authoritatively absent (both KeyAbsent), or a self-remount found the owner foreign, retired or at another life (also the admin pool of a SYSTEM CAS DROP POOL MEMBER run whose own claim was lost; its log line names the run). IdentityLost is a fail-loud TERMINAL state -- store-class access fails loud from this point and this pool's background threads (remount + GC) self-exit; there is no observer. Recover by restart or SYSTEM CAS FORGET; a matching-sentinel restore does not auto-revive.", ValueType::Number) \ M(CASDataRootVanished, "Counts CAS pools that entered a terminal Vanished lifecycle state (replaced / forgotten): the pool's data root was proven foreign (pool_id mismatch) or was decommissioned by SYSTEM CAS FORGET. The disk stays registered but store-class access fails loud with a typed error naming the sub-state; restart re-registers the name.", ValueType::Number) \ M(CASRefRecoveryEpochSealed, "Number of CAS ref epoch-seal transactions MINTED by a recovery compare-and-swap walk, one per dead writer epoch it closed. Each seal occupies the exact log key a dying predecessor's in-flight PUT would take, so growth tracks epoch transitions over touched namespaces (including burned, never-written epochs), not anomalies.", ValueType::Number) \ M(CASRefRecoveryEpochSealAdopted, "Number of dead CAS ref writer epochs a recovery compare-and-swap walk found ALREADY closed by a concurrent recoverer's seal and adopted. A peer that got there first is the designed outcome, not contention to alarm on.", ValueType::Number) \ @@ -1016,7 +1018,7 @@ The server successfully detected this situation and will download merged part fr M(S3CompleteMultipartUpload, "Number of S3 API CompleteMultipartUpload calls.", ValueType::Number) \ M(S3PutObject, "Number of S3 API PutObject calls.", ValueType::Number) \ M(S3GetObject, "Number of S3 API GetObject calls.", ValueType::Number) \ - M(S3SingleAttemptRetryConsultations, "Number of AWS SDK retry consultations refused by the single-attempt S3 retry profile. Non-zero means the SDK attempted a transparent retry on a write that must make exactly one HTTP attempt.", ValueType::Number) \ + M(S3SingleAttemptRetryConsultations, "Number of AWS SDK retry consultations refused by a single-attempt S3 client: the clone used by the single-attempt retry profile, or the clone that sends the object-creating requests of a write that forbids a resend. Non-zero means a request on such a client failed and was not resent.", ValueType::Number) \ \ M(DiskS3DeleteObjects, "Number of DiskS3 API DeleteObject(s) calls.", ValueType::Number) \ M(DiskS3CopyObject, "Number of DiskS3 API CopyObject calls.", ValueType::Number) \ diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp index f2fa931542431..9d8dead79945f 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.cpp @@ -802,10 +802,9 @@ SentinelProbeResult ObjectStorageBackend::probeSentinelUnder(const String & key, /// Settings for a genuine Native conditional write. Mark the request for the typed conditional /// dialect and, on a generation-token store, force a single PUT: GCS does not enforce the condition -/// on multipart completion. Blob publication never uses these settings; it remains an ordinary -/// unconditional multipart-capable write. CAS-mutable keys (shard manifests, gc/state, the registry) -/// also skip the racy post-upload existence/size check; a publish's manifest CAS was observed racing -/// the GC fence there. +/// on multipart completion. Blob publication uses `blobPublicationWriteSettings` instead. CAS-mutable +/// keys (shard manifests, gc/state, the registry) also skip the racy post-upload existence/size check; +/// a publish's manifest CAS was observed racing the GC fence there. WriteSettings ObjectStorageBackend::conditionalWriteSettings(size_t attempt_no) const { WriteSettings ws; @@ -875,6 +874,18 @@ std::expected ObjectStorageBackend::write( return emuWrite(key, bytes); } +/// On a content-ETag store a resent envelope that lands after GC deleted it restores the condemned +/// incarnation, and the exact-token delete of that incarnation then removes a live body. Retries belong +/// to the caller, which re-observes and publishes a fresh envelope. Requests keep the storage's own +/// request timeout, which a body upload needs. +WriteSettings ObjectStorageBackend::blobPublicationWriteSettings() const +{ + WriteSettings ws; + ws.object_storage_single_attempt_object_creation = true; + ws.object_storage_connect_timeout_cap_ms = connect_timeout_cap_ms; + return ws; +} + void ObjectStorageBackend::publish(const BlobPublishRequest & request, TransportAccess &) { if (const auto * streaming = std::get_if(&request.publication)) @@ -894,14 +905,14 @@ void ObjectStorageBackend::publish(const BlobPublishRequest & request, Transport return; } - /// Ordinary unconditional rewrite: default request mode, retry profile, and multipart policy. - /// In particular, generation stores are not restricted by the conditional single-PUT cap. + /// Unconditional rewrite: default request mode and multipart policy, so generation stores are + /// not restricted by the conditional single-PUT cap. auto out = object_storage->writeObject( StoredObject(request.destination_key), WriteMode::Rewrite, /*attributes=*/std::nullopt, DBMS_DEFAULT_BUFFER_SIZE, - WriteSettings{}); + blobPublicationWriteSettings()); out->write(streaming->fresh_envelope.data(), streaming->fresh_envelope.size()); blob_publication_detail::BlobPayloadCopyResult copy_result; try @@ -934,7 +945,7 @@ void ObjectStorageBackend::publish(const BlobPublishRequest & request, Transport ErrorCodes::NOT_IMPLEMENTED, "ObjectStorageBackend::publish: verbatim staged publication requires Native mode"); - WriteSettings write_settings; + WriteSettings write_settings = blobPublicationWriteSettings(); write_settings.object_storage_copy_mode = ObjectStorageCopyMode::NativeOnly; if (!object_storage->supportsCopyMode(write_settings.object_storage_copy_mode)) throw Exception( diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h index aef51c04816f7..f9fa31e55fbb9 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasObjectStorageBackend.h @@ -205,6 +205,8 @@ class ObjectStorageBackend final : public Backend /// physical-attempt count (see `TransportAccess::attemptNo`), carried into /// `object_storage_attempt_number` so the HTTP client sees a reissue as attempt >= 2. WriteSettings conditionalWriteSettings(size_t attempt_no) const; + /// Settings for an unconditional blob publication: each object-creating request goes out at most once. + WriteSettings blobPublicationWriteSettings() const; WriteSettings conditionalWriteSettingsForTest() const { return conditionalWriteSettings(/*attempt_no=*/1); } /// Override the emulated backend's wall clock for deterministic expiry tests. void setEmuNowNsForTest(uint64_t now_ns); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp index 625c04af8b41d..481910f1d91f7 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasProbe.cpp @@ -36,8 +36,8 @@ Removal removeOrReportDeleteMarker(CasOperation & op, const String & key, const "object VERSIONING enabled, and a content-addressed pool cannot run on a versioned bucket: " "every GC delete would archive a noncurrent version instead of reclaiming storage (the bucket " "grows forever), and the constantly-rewritten ref objects would pile up versions on every " - "commit. This is NOT ignorable and has no override. Use a bucket where versioning was NEVER " - "enabled — note that merely SUSPENDING versioning is not enough (deletes on a " + "commit. Use a bucket where versioning was NEVER enabled; skip_access_check skips this probe " + "but does not make a versioned bucket usable — note that merely SUSPENDING versioning is not enough (deletes on a " "versioning-suspended bucket still mint delete markers, so this probe will refuse again)"); } } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp index a1366b6d61cd6..9e96b1a33efba 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -278,6 +279,27 @@ bool isFirstAttemptFuseTimeout([[maybe_unused]] const std::exception & e, [[mayb #endif } +bool isStoreAnswer(const std::exception & e) +{ + const auto * db = dynamic_cast(&e); + return dynamic_cast(&e) && !(db && isDeterministicLocalFailure(db->code())); +} + +String describeRequestFailure(std::string_view verb, const String & key, const std::exception & e) +{ + const auto & poco = dynamic_cast(e); + String code; +#if USE_AWS_S3 + if (const auto * s3 = dynamic_cast(&e)) + code = s3->getExceptionName(); +#endif + if (const auto * db = dynamic_cast(&e); code.empty() && db) + code = String(ErrorCodes::getName(db->code())); + if (code.empty()) + code = poco.name(); + return fmt::format("{} {}: {} {} {}", verb, key, isDefinitelyRefusedWrite(e) ? "refused" : "transport", code, poco.message()); +} + CasRequests::CasRequests(BackendPtr backend_, Fence fence_, std::function now_ms_, std::function sleep_ms_, CasHotKeys * hot_keys_) @@ -871,14 +893,23 @@ std::optional CasOperation::gatedPause(uint64_t pause_ms, uint32_t return std::nullopt; } +uint64_t CasOperation::reissuePause(const WriteState & state, uint64_t unspaced_ms) const +{ + if (!state.spacing) + return unspaced_ms; + return Retry::pauseAfterAttempt(state.spacing->spacing_ms, state.spacing->attempt_started_ms, owner.now_ms()); +} + std::optional CasOperation::pauseAndReissue(WriteState & state, const Retry::Bound & bound) { - return gatedPause(Retry::backoff(++state.reissues), 2, state, bound, detail::recordReissue, /*should_sleep=*/true); + const uint64_t pause_ms = reissuePause(state, Retry::backoff(++state.reissues)); + return gatedPause(pause_ms, 2, state, bound, detail::recordReissue, /*should_sleep=*/true); } std::optional CasOperation::pauseForConflict(WriteState & state, const Retry::Bound & bound) { - return gatedPause(Retry::conflictBackoff(), 2, state, bound, detail::recordConflictPause, /*should_sleep=*/true); + return gatedPause(reissuePause(state, Retry::conflictBackoff()), 2, state, bound, detail::recordConflictPause, + /*should_sleep=*/true); } /// A flat pause before reissuing an attempt whose failure text named a failed connection. @@ -886,7 +917,7 @@ static constexpr uint64_t kConnectHintPauseMs = 50; std::optional CasOperation::pauseFlat(WriteState & state, const Retry::Bound & bound) { - return gatedPause(kConnectHintPauseMs, 2, state, bound, detail::recordReissue, /*should_sleep=*/true); + return gatedPause(reissuePause(state, kConnectHintPauseMs), 2, state, bound, detail::recordReissue, /*should_sleep=*/true); } std::optional CasOperation::reissueAtOnce(WriteState & state, const Retry::Bound & bound) @@ -921,9 +952,13 @@ WriteResult CasOperation::writeLoop(const String & key, const String & bytes, co if (!fits(reservation, bound)) return gaveUp(GaveUp::Why::Deadline, sourceFor(bound), state); + if (policy.attempt_spacing_ms) + state.spacing = WriteState::Spacing{*policy.attempt_spacing_ms, owner.now_ms()}; detail::recordAttempt(); ++state.attempts_sent; state.sent_any = true; + if (observer.on_write_attempt) + observer.on_write_attempt(state.attempts_sent); /// Disengaged means the attempt threw: its fate is unproven, and nothing may be read out of it. std::optional> outcome; @@ -952,6 +987,8 @@ WriteResult CasOperation::writeLoop(const String & key, const String & bytes, co { if (isDeterministicLocalFailure(e.code())) throw; + if (observer.on_failure) + observer.on_failure("PUT", key, e); /// ONE refresh per call, only for the class a credential could explain, and only when a /// reissue could sign with what it installs. So an oversized entity never triggers a /// re-acquisition, a denial fresh credentials do not fix is refused on the second look @@ -1008,6 +1045,8 @@ WriteResult CasOperation::writeLoop(const String & key, const String & bytes, co /// still have landed, so `outcome` stays disengaged and the read below settles it. if (!dynamic_cast(&e)) throw; + if (observer.on_failure) + observer.on_failure("PUT", key, e); } if (outcome && outcome->has_value()) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h index 3195d35bb8965..92ebe09b92635 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRequests.h @@ -60,6 +60,10 @@ bool isFirstAttemptFuseTimeout(const std::exception & e, size_t attempt_no); /// same failure and buries the root cause behind a retryable exception. The set is `LOGICAL_ERROR`, /// `NOT_IMPLEMENTED`, `BAD_ARGUMENTS` and `CORRUPTED_DATA`. bool isDeterministicLocalFailure(int code); +/// An exception the store or the transport raised, as opposed to a local fault. +bool isStoreAnswer(const std::exception & e); +/// `VERB key: refused|transport code message`; `code` is the store's error name when it sent one. +String describeRequestFailure(std::string_view verb, const String & key, const std::exception & e); /// Throw the recoverable "CAS write could not be committed, retry later" condition. /// @@ -157,6 +161,15 @@ struct ListPage /// The walk's callback: FALSE stops the walk. using ListedKeyFn = std::function; +/// Live progress of one operation, called on its thread inside the engine's retry loops; must not throw. +struct RequestObserver +{ + /// A conditional write is about to send its `attempts_sent`-th physical attempt. + std::function on_write_attempt; + /// A conditional `PUT` or a body `GET` failed with a store answer (`isStoreAnswer`). + std::function on_failure; +}; + namespace detail { /// The engine's per-attempt counters, behind functions so the header need not declare the events. @@ -281,6 +294,7 @@ class CasOperation /// The verdict point: is this operation still admitted? For the sites that guard a decision rather /// than a request. bool admitted() const { return gate(0) == Gate::Ok; } + void observeRequests(RequestObserver observer_) { observer = std::move(observer_); } /// The write lane for keys several writers of this pool share. CasHotKeys & hotKeys() const { return *owner.hot_keys; } @@ -365,6 +379,13 @@ class CasOperation Observation last_seen = NotObserved{}; uint32_t reissues = 0; bool refresh_attempted = false; + /// Under `Retry::attempt_spacing_ms`: that spacing, and when the latest attempt was sent. + struct Spacing + { + uint64_t spacing_ms = 0; + uint64_t attempt_started_ms = 0; + }; + std::optional spacing; }; /// Why a read-class request stopped without an answer. Every give-up below throws the same @@ -430,7 +451,8 @@ class CasOperation std::optional gatedPause(uint64_t pause_ms, uint32_t envelopes, WriteState & state, const Retry::Bound & bound, void (*record)(), bool should_sleep); /// Admission, then the jittered sleep. A value means the call ended during it; nullopt means the - /// caller may send another attempt. + /// caller may send another attempt. Under `WriteState::spacing` this and the two siblings below + /// sleep `reissuePause` instead. std::optional pauseAndReissue(WriteState & state, const Retry::Bound & bound); /// The sibling for a clean lost race: the same admission and the same reservation, a flat /// `Retry::conflictBackoff` sleep, and `state.reissues` untouched, so a transport fault that follows @@ -443,6 +465,8 @@ class CasOperation /// sleep at all, and `state.reissues` untouched -- the fuse is a connection-quality answer about a /// fresh connection, not a store fault, so nothing here is paced against it. std::optional reissueAtOnce(WriteState & state, const Retry::Bound & bound); + /// The pause before a write reissue: `unspaced_ms`, or `Retry::pauseAfterAttempt` under `state.spacing`. + uint64_t reissuePause(const WriteState & state, uint64_t unspaced_ms) const; /// `sleep_ms` plus `envelopes` attempt reservations, saturating. uint64_t reservedFor(uint64_t sleep_ms, uint32_t envelopes) const; @@ -470,6 +494,7 @@ class CasOperation /// Written immediately before a read-class give-up throws, cleared and read only by the resolve /// read that swallows it. Every other caller lets the exception carry the verdict. std::optional last_read_stop; + RequestObserver observer; }; template @@ -477,6 +502,7 @@ auto CasOperation::readLoop(std::string_view verb, const String & subject, const const Retry::Bound & bound, Fn && once) { bool refresh_attempted = false; + uint64_t attempt_started_ms = 0; /// Two counters, deliberately kept separate: `attempt_no` is the PHYSICAL attempt count handed to /// the transport (so a reissue is seen as attempt >= 2); `ordinary_reissues` is the /// exponential-backoff index. They advance together on an ordinary failure, but the first-attempt @@ -494,6 +520,8 @@ auto CasOperation::readLoop(std::string_view verb, const String & subject, const if (!fits(reservation, bound)) giveUpReadDeadline(verb, subject, bound, attempt_no - 1); + if (policy.attempt_spacing_ms) + attempt_started_ms = owner.now_ms(); detail::recordAttempt(); try { @@ -501,6 +529,8 @@ auto CasOperation::readLoop(std::string_view verb, const String & subject, const } catch (const std::exception & e) { + if (observer.on_failure && verb == "read" && isStoreAnswer(e)) + observer.on_failure("GET", subject, e); bool refreshed = false; if (refreshAndClassifyReadFault(e, refresh_attempted, refreshed)) throw; @@ -534,7 +564,10 @@ auto CasOperation::readLoop(std::string_view verb, const String & subject, const } } - const uint64_t pause_ms = Retry::backoff(++ordinary_reissues); + ++ordinary_reissues; + const uint64_t pause_ms = policy.attempt_spacing_ms + ? Retry::pauseAfterAttempt(*policy.attempt_spacing_ms, attempt_started_ms, owner.now_ms()) + : Retry::backoff(ordinary_reissues); const uint64_t needed = reservedFor(pause_ms, 1); switch (gate(needed)) { diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRetry.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRetry.cpp index c7b4c27215b53..f7dfde7d3c905 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRetry.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRetry.cpp @@ -17,6 +17,22 @@ uint64_t Retry::backoff(uint32_t attempt) return thread_local_rng() % (ceiling + 1); /// full jitter: uniform(0, ceiling) } +uint64_t Retry::spacedPause(uint64_t spacing_draw_ms, uint64_t attempt_started_ms, uint64_t now_ms) +{ + const uint64_t taken_ms = now_ms > attempt_started_ms ? now_ms - attempt_started_ms : 0; + return taken_ms >= spacing_draw_ms ? 0 : spacing_draw_ms - taken_ms; +} + +uint64_t Retry::drawSpacing(uint64_t attempt_spacing_ms_) +{ + const uint64_t spread = attempt_spacing_ms_ / 5; + const uint64_t low = attempt_spacing_ms_ - spread; + const uint64_t high = attempt_spacing_ms_ > std::numeric_limits::max() - spread + ? std::numeric_limits::max() + : attempt_spacing_ms_ + spread; + return low + thread_local_rng() % (high - low + 1); +} + Retry::Bound Retry::bind(uint64_t now_ms) const { const uint64_t own_deadline_ms = policy_deadline_ms diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRetry.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRetry.h index de235e8fd3faf..86336201c53e2 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRetry.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasRetry.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include namespace DB::Cas @@ -23,6 +24,12 @@ struct Retry /// iterations a fresh window. Empty for a single verb, which gets its window from where it is /// called. std::optional policy_deadline_ms = std::nullopt; + /// When set, a reissue in `CasOperation`'s write and read loops waits until a draw from [0.8, 1.2] + /// of this has passed since the failed attempt started, instead of the growing backoff. A + /// first-attempt fuse timeout is still reissued at once. + std::optional attempt_spacing_ms = std::nullopt; + + static constexpr uint64_t kDefinitiveAttemptSpacingMs = 1'000; /// Full jitter: uniform(0, min(5000, 200 << (attempt-1))) milliseconds. `attempt` is 1-based; /// `attempt == 0` returns 0. @@ -53,6 +60,23 @@ struct Retry } /// The standard policy, but at most one attempt is ever sent. static Retry once() { return {.window_ms = 90'000, .lease_deadline_ms = std::nullopt, .single_attempt = true}; } + /// No window and no lease bound: the operation ends on a definitive answer, or when the fence or the + /// caller's liveness refuses. Reissues are spaced by `kDefinitiveAttemptSpacingMs`. + static Retry untilDefinitive() + { + return {.window_ms = std::numeric_limits::max(), .lease_deadline_ms = std::nullopt, + .single_attempt = false, .attempt_spacing_ms = kDefinitiveAttemptSpacingMs}; + } + /// `spacing_draw_ms` minus what the failed attempt took, not below 0. A sample before the start + /// counts as no time taken. + static uint64_t spacedPause(uint64_t spacing_draw_ms, uint64_t attempt_started_ms, uint64_t now_ms); + /// A uniform draw in [0.8, 1.2] of `attempt_spacing_ms_`, saturating at the top of the range. + static uint64_t drawSpacing(uint64_t attempt_spacing_ms_); + /// The pause before the next attempt under `attempt_spacing_ms`: `spacedPause` of a fresh draw. + static uint64_t pauseAfterAttempt(uint64_t attempt_spacing_ms_, uint64_t attempt_started_ms, uint64_t now_ms) + { + return spacedPause(drawSpacing(attempt_spacing_ms_), attempt_started_ms, now_ms); + } /// This policy, made single-attempt. A frozen loop policy keeps its absolute deadline through it, /// which is what lets a loop send an unrepeatable request under the same bound as the rest. diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasWriteResult.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasWriteResult.h index 72679c54b64a0..722c6c2aa7449 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasWriteResult.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Backend/CasWriteResult.h @@ -121,7 +121,9 @@ inline std::optional orThrow(WriteResult && result, std::string_view what) case GaveUp::Why::Deadline: throwCasWriteRetryLater(fmt::format("{}: gave up at the {} deadline after {} attempt(s)", what, g.deadline_source == GaveUp::Source::Lease ? "lease" : "policy", g.sent_any ? "one or more" : "zero")); case GaveUp::Why::Unresolved: - throwCasWriteRetryLater(fmt::format("{}: the write is unresolved (sent, resolve read found {})", what, renderObservation(g.last_seen))); + throwCasWriteRetryLater(g.sent_any + ? fmt::format("{}: the write is unresolved (sent, resolve read found {})", what, renderObservation(g.last_seen)) + : fmt::format("{}: no write was sent; a read of the current object did not complete ({})", what, renderObservation(g.last_seen))); } UNREACHABLE(); }}, result); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.cpp index 6b1a7c60c7783..bb28ae3e1c052 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.cpp @@ -467,8 +467,8 @@ CasLifecycleSnapshot ContentAddressedMetadataStorage::lifecycleSnapshot() const } const Cas::Pool::LifecycleSnapshot ps = pool->lifecycleSnapshot(); - snap.lifecycle = casLifecycleToString(ps.lifecycle); - snap.reason = casLifecycleReasonWord(ps.lifecycle); + snap.lifecycle = ps.writes_paused ? "not_live" : casLifecycleToString(ps.lifecycle); + snap.reason = ps.writes_paused ? "lease_expired" : casLifecycleReasonWord(ps.lifecycle); snap.detail = ps.detail; snap.since = ps.since; return snap; @@ -895,7 +895,8 @@ void ContentAddressedMetadataStorage::startup() { Cas::sweepOwnMountStaging( *object_storage, - physicalKey(view.pool_prefix + "/staging/" + server_root_id) + "/"); + physicalKey(Cas::Layout(view.pool_prefix).stagingPrefix(server_root_id)), + pool->liveWriterEpoch()); } /// The background GC scheduler runs only on the disk-factory path (context non-null) and when @@ -1356,7 +1357,12 @@ String ContentAddressedMetadataStorage::stagingKeyPrefix() const { /// This is the writer-owned `staging//` subtree. `store` throws `INVALID_STATE` /// when no pool is published; every writer caller runs after `startup`. - return physicalKey(store()->poolConfig().pool_prefix + "/staging/" + server_root_id); + return physicalKey(store()->layout().stagingPrefix(server_root_id)); +} + +String ContentAddressedMetadataStorage::stagingKey(uint64_t writer_epoch, const String & leaf) const +{ + return physicalKey(store()->layout().stagingKey(server_root_id, writer_epoch, leaf)); } /// ==== namespace mapping ==== diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.h index a3420d6a7ca05..906ce706b6d8a 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedMetadataStorage.h @@ -95,13 +95,17 @@ enum class CasOpAdmission : uint8_t /// - `lifecycle` — one of `live` / `not_live` / `identity_lost` / `vanished` (a live pool), or /// `constructing` / `shutdown` (no pool published). /// - `reason` — the ENUM-CLEAN sub-state word: `replaced` / `forgotten` for a -/// `vanished` pool, empty otherwise. Kept a small closed vocabulary so a downstream +/// `vanished` pool, `lease_expired` for a `not_live` row whose pool is `Live` but +/// whose lease has too little time left to admit a write, empty otherwise. Kept a +/// small closed vocabulary so a downstream /// `lifecycle || '(' || reason || ')'` yields e.g. exactly `vanished(forgotten)` — /// the [D5] free text lives in `detail`, never here. /// - `detail` — the full [D5] reason text naming the actual failure (the replaced /// diagnosis, the timestamped `FORGET` message, or the identity-loss message); spec §1 -/// requires it appear verbatim in the snapshot. Empty while `live` and for a null pool. -/// - `since` — wall-clock second the current non-`live` state was entered; 0 while `live`/no pool. +/// requires it appear verbatim in the snapshot. Empty while `live`, for +/// `lease_expired`, and for a null pool. +/// - `since` — wall-clock second the current non-`live` state was entered (for +/// `lease_expired`, when writes paused); 0 while `live`/no pool. /// - `pool_id` — last-known pool UUID (empty before the first `startup`); the disk stays /// introspectable under its identity even once the pool is gone. /// - `server_root_id` — this server's node-local root id owning the mount slot. @@ -398,9 +402,10 @@ class ContentAddressedMetadataStorage final : public IMetadataStorage, public IC /// Returns the underlying object storage for an S3 staging writer. It is meaningful only when /// `stagingBackend` is `S3`. const ObjectStoragePtr & objectStorage() const { return object_storage; } - /// Returns the physical `pool_prefix/staging/server_root_id` prefix for this mount's staging - /// area. Callers append a unique leaf. + /// The physical `Cas::Layout::stagingPrefix` of this mount's staging area, with a trailing `/`. String stagingKeyPrefix() const; + /// The physical `Cas::Layout::stagingKey` of one staged upload of this mount. + String stagingKey(uint64_t writer_epoch, const String & leaf) const; /// Bytes that live INSIDE pool metadata rather than as their own object: an Inline-placement /// manifest tree entry, or a verbatim namespace file. nullopt = the path is blob-backed (a real diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp index ea2cef27fe60a..f403058243c23 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedSettings.cpp @@ -28,6 +28,10 @@ namespace ErrorCodes namespace { +/// Observers wait for the lease a holder declares, so a long lease would block reclaim of a silently +/// dead member for that long. The configured TTL is clamped here, before anything derives from it. +constexpr UInt64 max_mount_lease_ttl_ms = 600'000; + /// Poco renders repeated XML elements as `name`, `name[1]`, `name[2]` -- the convention /// `StorageURL` and `HTTPDictionarySource` both handle when reading repeated `
` elements. /// A key of ours that appears twice must be recognised by its base name rather than passed over as @@ -253,6 +257,14 @@ void ContentAddressedSettings::validate() "content_addressed disk: cas_mount_lease_ttl_ms must be >= 1 (got {})", settings[ContentAddressedSetting::mount_lease_ttl_ms].value); + if (settings[ContentAddressedSetting::mount_lease_ttl_ms] > max_mount_lease_ttl_ms) + { + LOG_WARNING(getLogger("ContentAddressedSettings"), + "content_addressed disk: cas_mount_lease_ttl_ms={} exceeds the maximum of {} ms; using {} ms", + settings[ContentAddressedSetting::mount_lease_ttl_ms].value, max_mount_lease_ttl_ms, max_mount_lease_ttl_ms); + settings[ContentAddressedSetting::mount_lease_ttl_ms] = max_mount_lease_ttl_ms; + } + if (settings[ContentAddressedSetting::mount_renew_period_ms] == 0) throw Exception(ErrorCodes::BAD_ARGUMENTS, "content_addressed disk: cas_mount_renew_period_ms must be >= 1 (got {})", diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.cpp index 071c1bb047dd4..56d82d99e8634 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.cpp @@ -127,9 +127,8 @@ ContentAddressedTransaction::ContentAddressedTransaction(ContentAddressedMetadat ContentAddressedTransaction::~ContentAddressedTransaction() { - /// Always clean up pending staging (whether committed or not). On the success path - /// cleanupPendingTempFiles was already called at the end of commit(); this call is the defensive - /// backstop for aborted/exception-unwound transactions whose publishStaging never ran. + /// On abort this is the cleanup of local and S3 staging. After a commit it finds nothing left, + /// except when the commit published some parts and then threw. cleanupPendingTempFiles(); /// An uncommitted transaction's uploads become min_active_build_sequence-spared debris: abandon every @@ -209,37 +208,20 @@ void ContentAddressedTransaction::cleanupPendingTempFiles() noexcept std::error_code ec; std::filesystem::remove(pb.staging_key, ec); } - else if (committed) + else { - /// A successful commit deletes the S3 staging object - /// HERE — `committed` is only ever true when EVERY part's `publishStaging` ran to - /// completion (commit() sets it right before this call), which means every referenced - /// pending blob was already materialized (`PartWriteTxn::putBlob` → `publishBlob`) - /// or, for an orphaned pending blob (its entry removed by `unlinkFile`/`replaceFile` - /// before commit), was never going to be promoted at all — either way the staging object - /// is no longer needed as a re-readable publication source, so it is safe to reclaim now. - /// - /// An ABORTED/exception-unwound transaction (`committed == false`, including a partial - /// multi-part commit failure where an EARLIER part's blobs were already promoted) leaves - /// its S3 staging objects in place — `staging_key` is a remote object-storage key, never a - /// bare `fs::remove` target, and is reclaimed by the mount-lease-scoped sweeper - /// (`Cas::sweepOwnMountStaging`), never here. This mirrors the local path's own asymmetry: - /// `Local` staging is a private per-transaction scratch file removed unconditionally on - /// both commit and abort (nobody else can ever read it), whereas an `S3` staging object - /// is the sanctioned re-readable source for publication and must outlive an aborted - /// transaction so a later attempt (or the sweeper) can still account for it. + /// The S3 staging object is private to this transaction and nothing reads + /// `pending_blobs` after this call, so commit and abort delete it alike. An object that + /// outlived an abort would stay until the next mount of this `server_root_id`. try { metadata_storage.objectStorage()->removeObjectIfExists(StoredObject(pb.staging_key)); } catch (...) // NOLINT(bugprone-empty-catch) { - /// Best-effort (noexcept context): a stubborn delete just leaves debris for the - /// mount-lease sweeper to reclaim on a later mount. + /// Best-effort (noexcept context): a stubborn delete leaves debris for the mount-start sweep. } } - /// else: an S3-mode pending blob of an ABORTED transaction — intentionally left in place - /// (see above); the mount-lease sweeper (`Cas::sweepOwnMountStaging`) is its reclaimer. } st.pending_blobs.clear(); } @@ -950,8 +932,6 @@ std::unique_ptr ContentAddressedTransaction::writeFile( if (metadata_storage.stagingBackend() == Cas::StagingBackend::S3) { - const std::string staging_key = metadata_storage.stagingKeyPrefix() + "/" + getRandomASCIIString(32) + ".tmp"; - auto object_sink = metadata_storage.objectStorage()->writeObject(StoredObject(staging_key), WriteMode::Rewrite); /// Build the fixed-length CABL envelope header now (before /// the payload is streamed) so the staging object holds `[header][payload]` and the promote /// stays a verbatim server-side copy. The header carries a FRESH `incarnation_tag`; `build_id` @@ -961,11 +941,12 @@ std::unique_ptr ContentAddressedTransaction::writeFile( std::string envelope_header = buildS3StagingBlobHeader(*r); /// The staged upload becomes durable in `sink->finalize()`, outside this request contract by /// design, so its admission is re-checked there through the callback below, against the - /// generation captured HERE. `admit().generation()` reads a plain value off a temporary - /// `CasOperation` that does not outlive this statement -- it names the fence generation at - /// THIS instant, not a live operation the two calls share. + /// generation captured HERE. The epoch in the key and the generation the finalize rechecks are + /// one capture, so a remount between them cannot stage new-generation bytes under an old epoch. const Cas::PoolPtr pool = metadata_storage.store(); - const uint64_t admitted_generation = pool->mountRequests().admit().generation(); + const Cas::MountIncarnation incarnation = pool->captureMountIncarnation(); + const std::string staging_key = metadata_storage.stagingKey(incarnation.writer_epoch, getRandomASCIIString(32) + ".tmp"); + auto object_sink = metadata_storage.objectStorage()->writeObject(StoredObject(staging_key), WriteMode::Rewrite); return std::make_unique( std::move(object_sink), staging_key, @@ -979,7 +960,7 @@ std::unique_ptr ContentAddressedTransaction::writeFile( const Cas::BlobRef ref{hash_algo, Cas::codecFor(hash_algo).fromHex(hash_hex)}; stageBlobPartFile(route, ref, size, key, Cas::StagingBackend::S3); }, - [pool, admitted_generation] { pool->checkFenceOrThrow(admitted_generation); }); + [pool, generation = incarnation.fence_generation] { pool->checkFenceOrThrow(generation); }); } return std::make_unique( diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.h index 1d421c2d2f03d..67843d679b60f 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/ContentAddressedTransaction.h @@ -33,8 +33,8 @@ namespace DB /// /// Content blobs are first represented by local or S3 staging objects and are uploaded only during /// publication. Inline entries remain in the manifest tree when they fit `INLINE_CAP`. The -/// destructor removes private local staging, leaves aborted S3 staging for the mount-lease sweeper, -/// and abandons any open part builds; it never publishes an uncommitted ref. +/// destructor removes the transaction's local and S3 staging and abandons any open part builds; +/// it never publishes an uncommitted ref. class ContentAddressedTransaction : public IMetadataTransaction { public: @@ -150,9 +150,8 @@ class ContentAddressedTransaction : public IMetadataTransaction /// loop is idempotent (never re-publishes a staging). /// `staging_key` is either a private local temp path or an S3 staging object key returned by - /// `CaContentWriteBuffer` at finalize. The backend selects cleanup and read-your-writes: - /// local files are removed by this transaction, whereas aborted S3 objects must remain - /// available to the mount-lease sweeper and to recovery of the promote source. + /// `CaContentWriteBuffer` at finalize. The backend selects the read-your-writes source; both + /// kinds are removed by this transaction on commit and on abort. struct PendingBlob { Cas::BlobRef ref; std::string staging_key; uint64_t size = 0; Cas::StagingBackend backend = Cas::StagingBackend::Local; }; std::vector pending_blobs; /// Staged blobs uploaded after the manifest edge is precommitted. }; @@ -197,8 +196,8 @@ class ContentAddressedTransaction : public IMetadataTransaction /// Parses a disk path and maps a part-file path to its namespace/ref/file route. std::optional routeOf(const std::string & path) const; - /// Removes local staging files after commit or abort. On a successful commit it also removes - /// S3 staging objects; aborted S3 objects are intentionally retained for lease-scoped cleanup. + /// Removes the transaction's staging after commit or abort: local files and S3 staging objects + /// alike. S3 deletes are best-effort; a lost delete leaves debris for the mount-start sweep. void cleanupPendingTempFiles() noexcept; /// Uploads through `st.build` only the pending blobs still referenced by `st.entries`. An entry diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h index fff4ab578791d..493dc4f880d64 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasFormat.h @@ -45,7 +45,7 @@ enum class FormatId : uint16_t /// Value 15 is retired: the fold seal is the sole per-generation coverage record after the /// one-pass GC round. The following three classes are per-server-root mount-safety objects. Owner = 16, /// `cas_owner` anchor from server-root ID to server UUID. - ServerEpoch = 17, /// `cas_epoch` writer-epoch fence carrying `next_writer_epoch`. + ServerEpoch = 17, /// `cas_epoch` writer-epoch fence carrying `next_writer_epoch` and an optional `allocation_nonce`. MountLease = 18, /// Live `cas_mount_lease` object. /// These identifiers cover objects that were added to the registry after their initial codecs: /// the ref transaction log, complete ref snapshot, blob freshness sidecar, and GC heartbeat. diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.cpp index d3c92e767c12b..3263e6095e156 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.cpp @@ -345,4 +345,20 @@ void Layout::checkNamespace(const RootNamespace & ns) const } } +std::optional Layout::stagedEpochOf(std::string_view rest) +{ + const size_t slash = rest.find('/'); + if (slash == std::string_view::npos) + return rest.empty() ? std::nullopt : std::optional(0); + const std::string_view dir = rest.substr(0, slash); + const std::string_view leaf = rest.substr(slash + 1); + if (dir.empty() || leaf.empty() || leaf.find('/') != std::string_view::npos) + return std::nullopt; + uint64_t epoch = 0; + const auto [end, ec] = std::from_chars(dir.data(), dir.data() + dir.size(), epoch); + if (ec != std::errc{} || end != dir.data() + dir.size()) + return std::nullopt; + return epoch; +} + } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h index 856b623b99b65..84d6c2ea3d4ad 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasLayout.h @@ -414,6 +414,19 @@ class Layout /// the sole reclaimer of `staging/` debris. String blobsPrefix() const { return prefix + "/blobs/"; } + /// One mount's S3 staging area: `/staging//`. + String stagingPrefix(const String & server_root_id) const { return prefix + "/staging/" + server_root_id + "/"; } + /// A staged upload: `/`. The epoch lets a successor's sweep tell a + /// predecessor's leftovers from its own in-flight uploads. + String stagingKey(const String & server_root_id, uint64_t writer_epoch, const String & leaf) const + { + return stagingPrefix(server_root_id) + std::to_string(writer_epoch) + "/" + leaf; + } + /// The writer epoch of a key under `stagingPrefix`, given the part after the prefix: `/` parses, + /// a flat `` is epoch 0 (written before keys carried one), anything else belongs to another member + /// nested under this prefix. + static std::optional stagedEpochOf(std::string_view rest); + /// Per-server-root control subtree, keyed by the configured `server_root_id` /// (validated by `DB::Cas::validateServerRootId`). All four control objects live together under /// `/gc/server-roots//` so a server's mount-safety state is one subtree. diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp index f430d2e4d612a..31773bf010e9b 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.cpp @@ -18,11 +18,14 @@ namespace OwnerWire { constexpr WireKey server_uuid{"server_uuid"}; constexpr WireKey retired_at_ms{"retired_at_ms"}; + constexpr WireKey owner_life{"owner_life"}; + constexpr WireKey retirement_verified{"retirement_verified"}; } namespace ServerEpochWire { constexpr WireKey next_writer_epoch{"next_writer_epoch"}; + constexpr WireKey allocation_nonce{"allocation_nonce"}; } namespace MountLeaseWire @@ -60,6 +63,10 @@ String encodeOwner(const OwnerObject & o) writeHex128Field(out, OwnerWire::server_uuid, o.server_uuid, first); if (o.retired_at_ms) writeNumberField(out, OwnerWire::retired_at_ms, *o.retired_at_ms, first); + if (o.owner_life != 0) + writeU64StringField(out, OwnerWire::owner_life, o.owner_life, first); + if (o.retirement_verified) + writeBoolField(out, OwnerWire::retirement_verified, true, first); closeObject(out, first); writeChar('\n', out); return std::move(out).take(); @@ -86,6 +93,10 @@ OwnerObject decodeOwner(std::string_view data) } else if (key == OwnerWire::retired_at_ms) rt = r.readU64Number(); + else if (key == OwnerWire::owner_life) + o.owner_life = r.readU64String(); + else if (key == OwnerWire::retirement_verified) + o.retirement_verified = r.readBool(); else r.skipUnknown(key); } @@ -103,6 +114,8 @@ String encodeServerEpoch(const ServerEpoch & e) writeHeaderLine(out, FormatId::ServerEpoch); bool first = true; writeU64StringField(out, ServerEpochWire::next_writer_epoch, e.next_writer_epoch, first); + if (e.allocation_nonce) + writeHex128Field(out, ServerEpochWire::allocation_nonce, *e.allocation_nonce, first); closeObject(out, first); writeChar('\n', out); return std::move(out).take(); @@ -126,11 +139,13 @@ ServerEpoch decodeServerEpoch(std::string_view data) e.next_writer_epoch = r.readU64String(); saw = true; } + else if (key == ServerEpochWire::allocation_nonce) + e.allocation_nonce = r.readHex128(); else r.skipUnknown(key); } - if (!saw) - throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS server-epoch: missing next_writer_epoch"); + if (!saw || e.next_writer_epoch == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS server-epoch: missing or zero next_writer_epoch"); if (!body_in.eof() || !in.eof()) throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS server-epoch: trailing bytes"); return e; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.h index 8e0bd8118a1e8..e67df65000de8 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/CasServerRootFormats.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -26,18 +27,36 @@ namespace DB::Cas struct OwnerObject { UInt128 server_uuid{}; - /// Set when this identity was explicitly decommissioned by an operator (`CasDecommission`'s - /// final step tombstones the owner object in place rather than deleting it, since its content + /// Set when this identity was explicitly decommissioned by an operator (`CasDecommission` + /// tombstones the owner object in place rather than deleting it, since its content /// alone cannot distinguish decommissioned debris from a legitimate successor's live anchor). /// A normal claim (`claimOwnerOrThrow`) must refuse to silently resume a tombstoned identity. std::optional retired_at_ms; + /// Every owner write sets this to the read value + 1, so two writes never carry the same bytes and an + /// `If-Match` against an older incarnation cannot land late. 0 = written before the field existed. + uint64_t owner_life = 0; + /// Set by the decommission run whose claim, tombstone and farewell all held; a later tombstone refresh + /// keeps it, a rollback drops it. + bool retirement_verified = false; }; +/// The owner of the next life after `current`. Every owner write after the first claim goes through it, so +/// the life only grows and no write repeats earlier bytes. +inline OwnerObject nextOwnerLife(const OwnerObject & current, std::optional retired_at_ms, bool retirement_verified) +{ + return OwnerObject{.server_uuid = current.server_uuid, .retired_at_ms = retired_at_ms, + .owner_life = current.owner_life + 1, .retirement_verified = retirement_verified}; +} + /// Durable counter state used to allocate unique writer epochs. The stored value is the next epoch /// to allocate; the caller CAS-writes a larger value and adopts the previous value for its mount. struct ServerEpoch { uint64_t next_writer_epoch = 0; + /// Random per allocation decision, so an unclear write is ours only if the stored bytes match + /// exactly: a twin of the same uuid bumping to the same value writes a different nonce. Absent in + /// bodies written by 26.6.4. + std::optional allocation_nonce = std::nullopt; }; /// The current liveness lease for one `(server_uuid, writer_epoch)` writer incarnation. The pool @@ -53,11 +72,24 @@ struct MountLease uint64_t started_at_ms = 0; uint64_t seq = 0; uint64_t expires_at_ms = 0; - uint64_t min_active_build_sequence = 0; /// UINT64_MAX = retired (farewell) + uint64_t min_active_build_sequence = 0; /// UINT64_MAX = clean release (farewell); not an owner retirement bool gc_fenced = false; /// GC fence-out of an expired lease; terminal /// One holder-originated body identity. Every physical retry of that one logical write reuses /// it; a GC fence copies the observed value while every successor holder body mints a new one. UInt128 write_attempt_id{}; + + /// The holder released the slot cleanly (its farewell): the `UINT64_MAX` sentinel in the build + /// watermark. Independent of `gc_fenced`, which callers check first. + bool isCleanRelease() const { return min_active_build_sequence == std::numeric_limits::max(); } + bool isHeldBy(UInt128 uuid, uint64_t epoch) const { return server_uuid == uuid && writer_epoch == epoch; } + /// What a fence writes over this body: the same holder, `gc_fenced`, one `seq` later. + MountLease fencedCopy() const + { + MountLease fenced = *this; + fenced.gc_fenced = true; + fenced.seq += 1; + return fenced; + } }; /// Encode the owner anchor as canonical text with the `cas_owner` header and a final newline. The @@ -66,8 +98,8 @@ struct MountLease /// decisions belong to the caller that coordinates the server-root object. String encodeOwner(const OwnerObject & o); -/// Decode an owner anchor, requiring its `server_uuid` field, tolerating an absent optional `retired_at_ms` retirement -/// timestamp, and rejecting bytes after the body line. Unknown JSON fields are skipped for +/// Decode an owner anchor, requiring its `server_uuid` field, tolerating absent `retired_at_ms`, `owner_life` and +/// `retirement_verified`, and rejecting bytes after the body line. Unknown JSON fields are skipped for /// forward-compatible reads; malformed input, a missing `server_uuid`, and trailing data throw /// `CORRUPTED_DATA`. OwnerObject decodeOwner(std::string_view data); @@ -77,8 +109,9 @@ OwnerObject decodeOwner(std::string_view data); /// codec. String encodeServerEpoch(const ServerEpoch & e); -/// Decode the epoch counter, requiring its `next_writer_epoch` field and rejecting bytes after the body line. -/// Unknown JSON fields are skipped for forward-compatible reads; malformed input, a missing `next_writer_epoch`, +/// Decode the epoch counter, requiring its `next_writer_epoch` field, tolerating an absent `allocation_nonce`, and +/// rejecting bytes after the body line. +/// Unknown JSON fields are skipped for forward-compatible reads; malformed input, a missing or zero `next_writer_epoch`, /// and trailing data throw `CORRUPTED_DATA`. ServerEpoch decodeServerEpoch(std::string_view data); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md index 03dc8f4f44c10..d615098337a95 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Formats/README.md @@ -32,7 +32,7 @@ trailer, followed by a banner-framed raw payload zone for inline file bytes. | `gc/gen//attempt//outcomes/…​.zst` | outcome log (`kind`, `outcome`) | `CasGcOutcomesFormat` (`.zst`) | GC | | `gc/gen//attempt//fold_seal` | fold seal (deterministic; `generation`/`parent_generation`, `kind`: `ref_life`/`blob_run`/`condemned`) | `CasFoldSealFormat` | GC | | `gc/gen//…​/runs` | GC source-edge record-stream runs (`ref`, `src`, `mark`; condemned: `pending`, `size`, `condemn_round`, `confirmed`) | `CasRecordStreamFormat` | GC | -| `gc/server-roots//{owner,epoch,mount}` | server-root singletons (`server_uuid`, optional `retired_at_ms`; `next_writer_epoch`; `server_uuid`, `writer_epoch`, `hostname`, `pid`, `started_at_ms`, `seq`, `expires_at_ms`, `min_active_build_sequence`, `gc_fenced`, `write_attempt_id`) | `CasServerRootFormats` | mount | +| `gc/server-roots//{owner,epoch,mount}` | server-root singletons (`server_uuid`, optional `retired_at_ms`, optional `owner_life` (u64 decimal string; absent in 26.6.4 bodies, read as 0; additive, no `v` bump), optional `retirement_verified` (bool; absent = false); `next_writer_epoch`, optional `allocation_nonce` (32 lowercase hex, drawn once per allocation decision; absent in 26.6.4 bodies, which still decode; additive ordinary key, no `v` bump); `server_uuid`, `writer_epoch`, `hostname`, `pid`, `started_at_ms`, `seq`, `expires_at_ms`, `min_active_build_sequence`, `gc_fenced`, `write_attempt_id`) | `CasServerRootFormats` | mount | | `roots/…` | raw passthrough (verbatim) | — (never interpreted) | upper layers | ## Codec table diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp index 368c2cf1379d2..d95e8a3f78405 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.cpp @@ -629,7 +629,10 @@ void Gc::applyRedeleteOutcome(RedeleteRoundContext & ctx, const RetiredEntry & e /// no reason (the meta is advisory, but there is no reason to touch it on that path at all). if (io.del == Removal::Removed || io.del == Removal::Gone) { - meta_writer->scheduleConfirmedMetaDelete(entry.ref, entry.condemn_round); + /// The round before this one, not `entry.condemn_round`: the graduation gate re-stamps a marker it + /// refused at its own, later round. Deleting a marker is safe for writers that adopt only on `Clean`; + /// a 26.6.4 writer still backfills an absent marker and adopts. + meta_writer->scheduleConfirmedMetaDelete(entry.ref, ctx.new_round - 1); } } @@ -838,24 +841,19 @@ RoundReport Gc::runRegularRound(std::function on_lease_acquired, bool al } /// Etag-guarded fence-out of dead mounts (liveness only — graduation itself paces on GC - /// rounds via `new_round`, not on heartbeat acks). Fencing no longer trusts a predecessor's stamped - /// `expires_at_ms` against our wall clock — it - /// fences ONLY once `mount_obs` has watched the mount's write-token hold unchanged for the full - /// threshold on THIS leader's own monotonic clock (shares `claimMountAwaitingExpiry`'s - /// `TTL + Drift` formula for a mount's own reopen wait, but with the full renewal period as the - /// cadence term below instead of half of it, so the two thresholds are close, not identical). + /// rounds via `new_round`, not on heartbeat acks). A slot is fenced only after `mount_obs` has + /// watched its token hold unchanged on this leader's own monotonic clock for + /// `mountObservationThresholdMs` over the longer of this server's lease and the one the slot + /// declares, with the full renewal period as the step. const uint64_t ttl_ms = static_cast(store->poolConfig().mount_lease_ttl_ms.count()); - /// The formula is shared with `claimMountAwaitingExpiry` via - /// `mountObservationThresholdMs` -- see its doc comment (CasServerRoot.h). - const uint64_t stable_threshold_ms = mountObservationThresholdMs( - ttl_ms, static_cast(store->poolConfig().mount_renew_period.count())); + const uint64_t renew_period_ms = static_cast(store->poolConfig().mount_renew_period.count()); /// PHASE 3/18 `heartbeat_floor`: one LIST of `gc/server-roots/`, one GET per mount slot, and a fence /// PUT per newly-fenced mount. { GcPhaseTimer t(phase_sink, "heartbeat_floor"); const HeartbeatFloor floor = computeHeartbeatFloor(op, layout, now_ms_fn(), mono_ms_fn, - stable_threshold_ms, mount_obs); + ttl_ms, renew_period_ms, mount_obs, lease_mismatch_warned); report.fence_outs = floor.fenced_now; if (floor.fenced_now > 0) ProfileEvents::increment(ProfileEvents::CASGCHeartbeatFenceOuts, floor.fenced_now); @@ -4901,11 +4899,8 @@ RebuildReport Gc::rebuildBaseline(bool force) /// returned classification counts are not needed for the round mint — graduation paces on rounds). /// Use the same threshold/`mount_obs` as the regular round. const uint64_t ttl_ms = static_cast(store->poolConfig().mount_lease_ttl_ms.count()); - /// Share the identical formula with `claimMountAwaitingExpiry` via - /// `mountObservationThresholdMs` -- see its doc comment (CasServerRoot.h). - const uint64_t stable_threshold_ms = mountObservationThresholdMs( - ttl_ms, static_cast(store->poolConfig().mount_renew_period.count())); - computeHeartbeatFloor(op, layout, now_ms_fn(), mono_ms_fn, stable_threshold_ms, mount_obs); + const uint64_t renew_period_ms = static_cast(store->poolConfig().mount_renew_period.count()); + computeHeartbeatFloor(op, layout, now_ms_fn(), mono_ms_fn, ttl_ms, renew_period_ms, mount_obs, lease_mismatch_warned); /// Retired-in-snapshot: the rebuilt seal's `condemned_summary` must be TOTAL over gc_shards so a /// subsequent regular round reads graduation/carry decisions zero-I/O off it (and its `carryParentRefs` diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h index cd3de5f2d48bb..3ffb51d180724 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGc.h @@ -1015,6 +1015,9 @@ class Gc /// leader (after a steal, or a process restart) starts empty, which only delays fencing an /// already-dead mount by one extra round (safe: never fences early). MountObservationMap mount_obs; + /// Srids `computeHeartbeatFloor` already warned about for a lease length different from ours. + /// In-memory: a new leader names each mismatched holder once more. + std::set lease_mismatch_warned; /// Throttle for the orphan sweep's retention rollup. The premise retains on any pass whose epochs are /// not closed-and-folded, so an unconditional `LOG_INFO` would repeat the same sentence for as long as diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMetaWriter.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMetaWriter.cpp index 62f49d74df883..e5be358883fbd 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMetaWriter.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMetaWriter.cpp @@ -33,9 +33,8 @@ namespace /// - every GC write stamps the round of the attempt that issues it, and replaces an absent marker, `Clean` /// or an older-round `Condemned`; /// - graduation accepts only `Condemned` at the entry's round or newer, read in that round (`Gc::fold`); -/// - a delete job removes only `Condemned` at its entry's round or older, `If-Match` the etag it read. -/// While `gc/state` is readable, a later incarnation that can own a job is condemned at a larger round, so no job -/// removes its marker; a rebuild that regresses rounds breaks this. +/// - a delete job removes only `Condemned` from a round before the deleting one, `If-Match` the etag it read. +/// Such a job may remove a later incarnation's marker; that is safe because a writer adopts only on `Clean`. /// /// GC never writes `Clean`, not even on a spare: a deposed leader that cleared a spare's marker and then lost /// its round CAS would leave `Clean` over a still-condemned body, and a stale exact-token redelete would delete @@ -55,10 +54,10 @@ void writeCondemnedMeta(CasOperation & op, const Layout & layout, const BlobRef /// Rounds only grow while `gc/state` is readable, so a marker at a newer round guards a later incarnation /// and `Clean` means a writer displaced the body: both are left alone. `If-Match` the etag read here, so a /// marker changed since the read is left alone too. -void deleteConfirmedMeta(CasOperation & op, const Layout & layout, const BlobRef & ref, uint64_t condemn_round) +void deleteConfirmedMeta(CasOperation & op, const Layout & layout, const BlobRef & ref, uint64_t newest_round) { const auto lm = loadMeta(op, layout, ref); - if (!lm || lm->meta.state != MetaState::Condemned || lm->meta.condemn_round > condemn_round) + if (!lm || lm->meta.state != MetaState::Condemned || lm->meta.condemn_round > newest_round) return; deleteMetaExact(op, layout, ref, lm->etag); } @@ -127,12 +126,12 @@ void GcMetaWriter::scheduleCondemnMarkerWrite(const BlobRef & ref, uint64_t cond }); } -void GcMetaWriter::scheduleConfirmedMetaDelete(const BlobRef & ref, uint64_t condemn_round) +void GcMetaWriter::scheduleConfirmedMetaDelete(const BlobRef & ref, uint64_t newest_round) { - submit([st = state, ref, condemn_round]() + submit([st = state, ref, newest_round]() { CasOperation op = st->store->openRequests().admit(); - deleteConfirmedMeta(op, st->store->layout(), ref, condemn_round); + deleteConfirmedMeta(op, st->store->layout(), ref, newest_round); }); } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMetaWriter.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMetaWriter.h index b57fbe46427b3..4e374b4291301 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMetaWriter.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasGcMetaWriter.h @@ -35,8 +35,9 @@ class GcMetaWriter void scheduleCondemnMarkerWrite(const BlobRef & ref, uint64_t condemn_round, uint64_t size); /// Drop the marker of a blob whose body is confirmed deleted or absent, but only while it is `Condemned` - /// at `condemn_round` or older: a newer marker, or `Clean`, belongs to a later incarnation. - void scheduleConfirmedMetaDelete(const BlobRef & ref, uint64_t condemn_round); + /// at `newest_round` or older; a newer marker and `Clean` stay. The marker removed may be a later + /// incarnation's, which is safe because a writer adopts only on `Clean`. + void scheduleConfirmedMetaDelete(const BlobRef & ref, uint64_t newest_round); /// Successful-path protocol barrier. Wait for every job scheduled so far, propagating any /// pool/framework exception recorded by `ThreadPool`; per-hash operation exceptions are caught diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.cpp index 8801ba25bac2f..1ef3ff7000ff5 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.cpp @@ -608,7 +608,8 @@ uint64_t sweepNamespace(Pool & store, const RootNamespace & ns, const BuildPrefi std::vector * warnings) { const Layout & layout = store.layout(); - CasOperation op = store.openRequests().admit(); + /// The decommission drain runs under the admin's lease; a fenced admin must stop deleting. + CasOperation op = store.mountRequests().admit(); if (!buildPrefixEligibleUnderFloorOn(op, layout, ns, prefix)) return 0; /// not eligible by the durable watermark fact — delete nothing (controls #8/#9) @@ -705,6 +706,7 @@ uint64_t sweepNamespace(Pool & store, const RootNamespace & ns, const BuildPrefi throw; warnings->push_back("CAS orphan sweep: " + listed.key + " delete failed: " + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + return op.admitted(); } return true; }, Retry::standard(), 1000, onGcEnumerationPage); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.h index 75646094f98f2..f96a20bef8cc1 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Gc/CasOrphanManifestSweep.h @@ -183,8 +183,8 @@ struct ManifestSweepResult /// /// `warnings`, when non-null, opts in to the decommission drain's tolerate-and-continue contract: a /// per-key transient failure (a thrown backend exception on `head`/`remove`) -/// is pushed onto `*warnings` and the sweep continues with the next key, instead of throwing out of -/// this call; likewise a protection-view-unavailable namespace (the pre-existing corrupt-snapshot skip +/// is pushed onto `*warnings` and the sweep continues with the next key while `store`'s mount still admits +/// requests, instead of throwing out of this call; likewise a protection-view-unavailable namespace (the pre-existing corrupt-snapshot skip /// below) also pushes a "cannot confirm emptiness" warning, not just a `LOG_WARNING`. `warnings == /// nullptr` (the default, every pre-existing caller) preserves the original behaviour exactly: a /// per-key failure propagates as an exception (fail-close default), and the protection-view skip is @@ -192,6 +192,8 @@ struct ManifestSweepResult /// normal "a fresh owner reclaimed it" race the periodic sweep expects, not a failure to warn about. /// This direct decommission path relies on the caller's held server-root claim/fence: while that claim /// is held, a same-server-root rebirth cannot become live between its catalog cut and exact-token delete. +/// Every request runs on `store`'s mount plane, so `store` must hold a writable mount; once its fence trips, +/// the remaining requests are refused. /// It still rejects every catalog cut with an ambiguous current life id before making any deletion decision. uint64_t sweepNamespace(Pool & store, const RootNamespace & ns, const BuildPrefix & prefix, std::vector * warnings = nullptr); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.cpp index 95c6fb7a6550e..e63b8df55dfa1 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -55,7 +56,7 @@ int64_t wallClockNowSeconds() CasMountRuntime::CasMountRuntime( BackendPtr backend_ptr_, CasRequests & mount_requests_, - CasRequests & farewell_requests_, + CasRequests & lease_requests_, const Layout & layout_, MountConfig config_, String server_root_id_, @@ -64,7 +65,7 @@ CasMountRuntime::CasMountRuntime( std::function remount_attempt_) : backend_ptr(std::move(backend_ptr_)) , mount_requests(mount_requests_) - , farewell_requests(farewell_requests_) + , lease_requests(lease_requests_) , layout(layout_) , config(std::move(config_)) , server_root_id(std::move(server_root_id_)) @@ -131,9 +132,10 @@ void CasMountRuntime::checkFenceOrThrow(uint64_t admitted_generation) const if (!mayMutate() || fenceGeneration() != admitted_generation) throwCasTransientUnavailable( fmt::format("content-addressed pool '{}'", server_root_id), - "mount fence tripped: the durable write is refused because this node no longer holds the mount " - "incarnation it was admitted under -- either a lease loss the disk auto-recovers from, or a " - "FORGET decommission / lost identity that does NOT recover; consult " + "mount fence tripped: the durable write is refused because the mount lease has too little time " + "left (writes are paused and resume once the lease renewal is confirmed), or because this node " + "no longer holds the mount incarnation it was admitted under -- either a lease loss the disk " + "auto-recovers from, or a FORGET decommission / lost identity that does NOT recover; consult " "system.cas_mounts for the disk's lifecycle before retrying"); } @@ -153,15 +155,49 @@ Fence::Admit CasMountRuntime::admit(uint64_t admitted_generation, uint64_t neede return Fence::Admit::Ok; } -bool CasMountRuntime::refAppendFenceOk() const +uint64_t CasMountRuntime::writeReservationMs() const { - /// Two envelopes' worth of room under the live generation -- a write and its settlement read, which - /// is what `writeLoop` reserves -- so a ref-log attempt is not started when it cannot plausibly - /// finish, safety margin included, before the lease expires. const uint64_t envelope_ms = cas_request_budget.attemptEnvelopeMs(); - const uint64_t needed_ms = envelope_ms > std::numeric_limits::max() / 2 - ? std::numeric_limits::max() : 2 * envelope_ms; - return admit(fenceGeneration(), needed_ms) == Fence::Admit::Ok; + return envelope_ms > std::numeric_limits::max() / 2 ? std::numeric_limits::max() : 2 * envelope_ms; +} + +bool CasMountRuntime::refAppendFenceOk() const +{ + /// A write and its settlement read, which is what `writeLoop` reserves, so a ref-log attempt is not + /// started when it cannot plausibly finish, safety margin included, before the lease expires. + return admit(fenceGeneration(), writeReservationMs()) == Fence::Admit::Ok; +} + +std::optional CasMountRuntime::writesPausedSinceBootMs() const +{ + if (lifecycle() != PoolLifecycle::Live || mount_fence.lost.load(std::memory_order_acquire) || refAppendFenceOk()) + return std::nullopt; + const uint64_t deadline = mount_fence.deadline_boot_ms.load(std::memory_order_acquire); + const uint64_t reservation_ms = writeReservationMs(); + const uint64_t closing_ms = reservation_ms > std::numeric_limits::max() - cas_request_budget.lease_safety_margin_ms + ? std::numeric_limits::max() + : reservation_ms + cas_request_budget.lease_safety_margin_ms; + return deadline > closing_ms ? deadline - closing_ms : 0; +} + +void CasMountRuntime::warnOnceIfWritesPaused() +{ + const std::optional paused_at = writesPausedSinceBootMs(); + if (!paused_at || writes_paused_warned.exchange(true, std::memory_order_relaxed)) + return; + std::optional last_error; + { + std::lock_guard lock(renewal_progress_mutex); + last_error = renewal_progress.last_error; + } + const uint64_t now = bootMsNow(); + /// Written at the first attempt after the pause began, so a renewal stuck in a request reports late; + /// the age says by how much. + LOG_WARNING(getLogger("CasPool"), + "CAS mount lease of '{}' under writer_epoch {} has too little time left for a write: writes are paused " + "since {} ms ago and resume once the renewal, which keeps retrying, is confirmed with enough lease left. " + "Last renewal error: {}", + server_root_id, liveWriterEpoch(), now > *paused_at ? now - *paused_at : 0, last_error.value_or("none")); } void CasMountRuntime::setMountDeadline(uint64_t deadline_boot_ms) @@ -174,6 +210,7 @@ void CasMountRuntime::armMountFence(UInt128 server_uuid, uint64_t writer_epoch, mount_fence.server_uuid = server_uuid; mount_fence.writer_epoch = writer_epoch; mount_fence.deadline_boot_ms.store(deadline_boot_ms, std::memory_order_release); + writes_paused_warned.store(false, std::memory_order_relaxed); /// A fresh lease incarnation is a fresh generation too: a durable-effect caller admitted under the /// PRIOR incarnation must re-check and abort rather than ride this re-arm through (rev.7 [C2]). fence_generation.fetch_add(1, std::memory_order_acq_rel); @@ -305,6 +342,11 @@ RenewalDriverState CasMountRuntime::DriverLease::finish( const MountRenewResult * result) { std::lock_guard lock(runtime.driver_mutex); + /// Published before the driver leaves the call, so a remount cannot arm a new epoch between this + /// renewal's commit and its deadline. The renewer cannot change meanwhile: `installRenewer` requires + /// a dormant or parked driver. + if (result && result->outcome == MountRenewOutcome::Committed) + runtime.publishRenewedDeadline(result->attempt_start_boot_ms); if (runtime.workers_stop_requested) runtime.renewal_driver_state = RenewalDriverState::Stopping; else if (active == RenewalDriverState::WorkerCall @@ -369,7 +411,7 @@ void CasMountRuntime::installRenewer( const std::function & now_ms) { auto replacement = std::make_unique( - mount_requests, farewell_requests, layout, server_root_id, our_uuid, writer_epoch, + mount_requests, lease_requests, layout, server_root_id, our_uuid, writer_epoch, config.mount_lease_ttl_ms, now_ms, [this] { return minActive(); }, [this](CasEvent e) { emitEvent(std::move(e)); }, @@ -421,8 +463,20 @@ uint64_t CasMountRuntime::startRenewer() return anchor; } +RenewalProgress CasMountRuntime::renewalProgress() const +{ + std::lock_guard lock(renewal_progress_mutex); + return renewal_progress; +} + MountRenewOperationEnvironment CasMountRuntime::renewalEnvironment(bool worker_call) { + uint64_t refusal_pause_ms = 0; + if (worker_call) + { + std::lock_guard lock(driver_mutex); + refusal_pause_ms = static_cast(std::max(0, renewal_period.count())); + } return MountRenewOperationEnvironment{ .boot_ms = [this] { return bootMsNow(); }, .live = [this, worker_call] @@ -430,6 +484,31 @@ MountRenewOperationEnvironment CasMountRuntime::renewalEnvironment(bool worker_c return config.renewal_live_for_test ? config.renewal_live_for_test() : renewalLive(worker_call); }, .cancelled = [this] { return renewalCancelled(); }, + .requests = RequestObserver{ + .on_write_attempt = [this](uint32_t attempts_sent) + { + warnOnceIfWritesPaused(); + ProfileEvents::incrementNoTrace(ProfileEvents::CASMountRenewalAttempts); + if (attempts_sent > 1) + ProfileEvents::incrementNoTrace(ProfileEvents::CASMountRenewalRetries); + std::lock_guard lock(renewal_progress_mutex); + renewal_progress.attempts_sent = attempts_sent; + }, + .on_failure = [this](std::string_view verb, const String & key, const std::exception & e) + { + /// Runs inside the engine's catch; a formatting failure must not change how the renewal ends. + try + { + String error = describeRequestFailure(verb, key, e); + std::lock_guard lock(renewal_progress_mutex); + renewal_progress.last_error = std::move(error); + } + catch (...) + { + } + }, + }, + .refusal_pause_ms = refusal_pause_ms, }; } @@ -454,7 +533,10 @@ bool CasMountRuntime::renewalCancelled() const void CasMountRuntime::sleepInterruptibly(uint64_t ms) { std::unique_lock lock(driver_mutex); - driver_cv.wait_for(lock, std::chrono::milliseconds(ms), [this] { return workers_stop_requested; }); + driver_cv.wait_for(lock, std::chrono::milliseconds(ms), [this] + { + return renewal_driver_state == RenewalDriverState::Stopping || renewal_driver_state == RenewalDriverState::ParkRequested; + }); } void CasMountRuntime::consumeRenewResult( @@ -465,13 +547,6 @@ void CasMountRuntime::consumeRenewResult( { /// Driver ownership has already been restored by `DriverLease::finish`; this is the single logical /// consumption boundary and it runs without `driver_mutex` or renewer access. - /// The physical counters come off the result rather than off a per-attempt callback, so they count - /// the same on every ending: a renewal that gave up still sent what it sent. - if (result.attempts_sent > 0) - { - ProfileEvents::incrementNoTrace(ProfileEvents::CASMountRenewalAttempts, result.attempts_sent); - ProfileEvents::incrementNoTrace(ProfileEvents::CASMountRenewalRetries, result.attempts_sent - 1); - } if (result.resolved_by_read) ProfileEvents::incrementNoTrace(ProfileEvents::CASMountRenewalResolved); @@ -481,17 +556,7 @@ void CasMountRuntime::consumeRenewResult( && result.deadline_source == GaveUp::Source::Lease) ProfileEvents::incrementNoTrace(ProfileEvents::CASMountRenewalDeadlineExceeded); - if (result.outcome == MountRenewOutcome::Committed) - { - const uint64_t ttl_ms = static_cast(config.mount_lease_ttl_ms.count()); - setMountDeadline( - result.attempt_start_boot_ms > std::numeric_limits::max() - ttl_ms - ? std::numeric_limits::max() - : result.attempt_start_boot_ms + ttl_ms); - reportMountRenewCompletion(result); - return; - } - if (result.outcome == MountRenewOutcome::NotAttempted) + if (result.outcome != MountRenewOutcome::Terminal) { reportMountRenewCompletion(result); return; @@ -521,6 +586,17 @@ void CasMountRuntime::consumeRenewResult( std::rethrow_exception(result.failure); } +void CasMountRuntime::publishRenewedDeadline(uint64_t attempt_start_boot_ms) +{ + const uint64_t ttl_ms = static_cast(config.mount_lease_ttl_ms.count()); + const uint64_t deadline = attempt_start_boot_ms > std::numeric_limits::max() - ttl_ms + ? std::numeric_limits::max() + : attempt_start_boot_ms + ttl_ms; + setMountDeadline(deadline); + if (!writesPausedSinceBootMs()) + writes_paused_warned.store(false, std::memory_order_relaxed); +} + uint64_t CasMountRuntime::renewRenewerOnce( AdmittedRenewerCall call, RenewalDriverState active, @@ -535,13 +611,26 @@ uint64_t CasMountRuntime::renewRenewerOnce( /// so it renews on the renewer's open plane: admitted under the mount fence it could only ever give /// up, and every remount would fail at this step. `RemountCall` is reached from /// `renewRenewerForRemountOnce` alone. - const MountRenewResult result = active == RenewalDriverState::RemountCall - ? call.renewer->renewForRemount(renewalEnvironment(worker_call)) - : call.renewer->renew(renewalEnvironment(worker_call)); + { + std::lock_guard lock(renewal_progress_mutex); + renewal_progress = RenewalProgress{.in_progress = true, .attempts_sent = 0, .last_error = std::nullopt}; + } + MountRenewResult result; + { + SCOPE_EXIT({ std::lock_guard lock(renewal_progress_mutex); renewal_progress.in_progress = false; }); + if (active == RenewalDriverState::RemountCall) + result = call.renewer->renewForRemount(renewalEnvironment(worker_call)); + else if (worker_call && !config.bounded_background_renewal) + result = call.renewer->renewUntilDefinitive(renewalEnvironment(worker_call)); + else + result = call.renewer->renew(renewalEnvironment(worker_call)); + } const RenewalDriverState destination = active == RenewalDriverState::WorkerCall ? RenewalDriverState::WorkerIdle : (active == RenewalDriverState::RemountCall ? RenewalDriverState::Parked : RenewalDriverState::Dormant); const RenewalDriverState returned_state = call.lease->finish(destination, &result); + if (config.renewal_finished_hook_for_test) + config.renewal_finished_hook_for_test(result); if (result.outcome == MountRenewOutcome::Terminal && config.renewal_terminal_deposited_hook_for_test) config.renewal_terminal_deposited_hook_for_test(); consumeRenewResult(result, active, returned_state, propagate_failure); @@ -940,7 +1029,7 @@ std::unique_lock CasMountRuntime::lockTerminalPublication() return lock; } -void CasMountRuntime::enterIdentityLost() +void CasMountRuntime::enterIdentityLost(const String & reason) { /// `TransientNotLive -> IdentityLost`, one way. The compare-exchange FROM `TransientNotLive` gives /// the brief's "from TransientNotLive only" precondition, idempotency (a second call finds the state @@ -975,12 +1064,11 @@ void CasMountRuntime::enterIdentityLost() ProfileEvents::increment(ProfileEvents::CASIdentityLost); LOG_WARNING(getLogger("CasPool"), - "Content-addressed pool '{}' entered IdentityLost: the pool sentinels (_pool_meta and the owner " - "anchor) are authoritatively absent (both KeyAbsent). This is a fail-loud TERMINAL state: " + "Content-addressed pool '{}' entered IdentityLost: {}. This is a fail-loud TERMINAL state: " "store-class access now fails loud and this pool's remount + GC threads self-exit. " "Recover by restart or SYSTEM CAS FORGET — a matching-sentinel restore does NOT " "auto-revive this disk.", - server_root_id); + server_root_id, reason); } void CasMountRuntime::enterVanished(PoolLifecycle which, const String & reason) @@ -1131,7 +1219,14 @@ void CasMountRuntime::finishTeardown(bool drained) if (!mount_renewer) return; - if (drained && mount_renewer->state() == MountLeaseRenewerState::Active) + if (drained && mount_renewer->state() == MountLeaseRenewerState::Active && writesPausedSinceBootMs()) + { + /// The farewell needs the lease room a pause no longer has; GC fences the slot after observing it. + LOG_INFO(getLogger("CasPool"), + "CAS mount lease of '{}' has too little time left for the farewell; the slot is left for observation", + server_root_id); + } + else if (drained && mount_renewer->state() == MountLeaseRenewerState::Active) { try { @@ -1146,7 +1241,7 @@ void CasMountRuntime::finishTeardown(bool drained) { LOG_WARNING( getLogger("CasPool"), - "CAS store shutdown with an unresolved ref-log PUT: skipping the clean-release marker; " + "CAS store shutdown with an unresolved write: skipping the clean-release marker; " "the next mount will treat this end as unclean"); } } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.h index 7f3ff1d76f36d..e5bb19a2c1e74 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasMountRuntime.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -31,8 +32,8 @@ using PartWriteTxnPtr = std::shared_ptr; /// - `Live` — the steady state; the mount lease is (or was last) held. /// - `TransientNotLive` — the lease was lost; access is uncertain and a self-remount retries. The §2 /// `Present`+identity-match recovery rule fires only from here (or `Live`). -/// - `IdentityLost` — the pool sentinels are authoritatively absent (both KeyAbsent): -/// fail-loud and TERMINAL (rev.8). The remount/GC workers self-exit; +/// - `IdentityLost` — the pool sentinels are authoritatively absent (both KeyAbsent), or a +/// remount found the owner at another life: fail-loud and TERMINAL (rev.8). The remount/GC workers self-exit; /// matching-sentinel reappearance does NOT auto-revive it ([D3]); recovery is a /// restart or `SYSTEM CAS FORGET`. /// - `Vanished*` — fully terminal truth: the data root was replaced by a foreign pool, or the @@ -108,6 +109,12 @@ struct MountConfig /// Test-only override of the renewal's liveness predicate, for exact pre/post-send gate /// interleavings. FALSE ends the renewal exactly as a lost fence does. std::function renewal_live_for_test = {}; + /// The decommission tool's pool: the background renewal stays bounded by the lease, as the startup + /// and remount re-anchors are, instead of retrying until a definitive answer. + bool bounded_background_renewal = false; + /// Deterministic test interposition on the renewing thread right after `DriverLease::finish` + /// returned driver ownership, before the result is consumed. Runs without `driver_mutex`. + std::function renewal_finished_hook_for_test = {}; }; /// Local, in-memory write fence. It is deliberately not checked by reading the object store for every @@ -134,6 +141,14 @@ struct MountFence std::atomic lost{false}; }; +/// The current logical renewal of the mount lease so far, or the last one once it ended. +struct RenewalProgress +{ + bool in_progress = false; + uint32_t attempts_sent = 0; + std::optional last_error; +}; + /// Owns the live writer-incarnation mechanics shared by the pool's mount and recovery orchestration: /// the `MountLeaseRenewer`, local `MountFence`, build watermark and in-flight build registry, /// `live_writer_epoch`, unclean-boundary marker, and both persistent workers. `Pool` retains the higher-level @@ -146,10 +161,11 @@ class CasMountRuntime public: CasMountRuntime( BackendPtr backend_ptr_, - /// The two planes the `MountLeaseRenewer` runs on: renewals under the mount fence, the farewell - /// on an open one. Owned by `Pool` and outliving this runtime. + /// The two planes the `MountLeaseRenewer` runs on: bounded renewals under the mount fence; the + /// claim, the remount re-anchor and the farewell on the open-fence lease plane. Owned by `Pool` + /// and outliving this runtime. CasRequests & mount_requests_, - CasRequests & farewell_requests_, + CasRequests & lease_requests_, const Layout & layout_, MountConfig config_, String server_root_id_, @@ -172,6 +188,8 @@ class CasMountRuntime /// Renew the merged mount heartbeat once, including its build-watermark floor. A read-only runtime /// has no renewer and fails with a logical exception rather than fabricating a heartbeat. void renewWatermarkOnce(); + /// Takes only its own lock, so a renewal stuck in a request never holds up the reader. + RenewalProgress renewalProgress() const; /// ---- local write fence ---- /// Return whether a mutable operation may start under the locally observed lease state. @@ -245,8 +263,9 @@ class CasMountRuntime /// (and the GC scheduler self-exits, through `Pool`) at its next boundary; there is no demoted observer. /// It deliberately does NOT publish the `vanished_intent` latch (which is reserved for the `Vanished*` /// idempotency/FORGET protocol); `remountTerminal` widens the worker-exit boundary to include it. + /// `reason` names the cause in that WARN: the pool sentinels are gone, or the owner moved to another life. /// Must be called under the caller's remount serialization (Pool::remount_mutex). - void enterIdentityLost(); + void enterIdentityLost(const String & reason); /// Test seam: force the lifecycle condition directly to `lc`, bypassing the natural transition /// preconditions (used by the operation-gate tests to pin each class × state cell without driving a /// full remount/erase sequence). For a `Vanished*` value it also latches `vanished_intent`, so the @@ -304,11 +323,15 @@ class CasMountRuntime return static_cast(lifecycle_since_wall_s.load(std::memory_order_acquire)); } - /// Extends `mayMutate` with a remaining-budget check. A ref-log attempt is refused unless the - /// current lease has room for its configured timeout and safety margin, so work is not started when - /// it cannot plausibly finish before the fence expires. + /// Extends `mayMutate` with a remaining-budget check: room for one write (two attempt envelopes) plus + /// the safety margin before the deadline. bool refAppendFenceOk() const; + /// When `refAppendFenceOk` stopped admitting writes under the current deadline, while the pool is + /// `Live` and the fence is not lost: the renewal keeps retrying and writes are paused. Empty while + /// writes are admitted. I/O-free. + std::optional writesPausedSinceBootMs() const; + /// TRUE once the pool has reached — or is being driven toward — a state on which the self-remount /// worker must stop: a published terminal `Vanished` intent (`vanished_intent` — set early by /// FORGET, or by a natural `enterVanished`, and already subsuming every settled `Vanished*` state since @@ -322,10 +345,11 @@ class CasMountRuntime || lifecycle() == PoolLifecycle::IdentityLost; } - /// The inter-attempt sleep the mount plane runs on. A plain sleep would hold a parked or stopping - /// renewal for the whole capped backoff; this one wakes on the same stop signal the workers watch. - /// It shortens a stop, not a fence loss: the fence cannot see a stop request, so a woken operation - /// still reissues unless its own liveness predicate refuses. + /// The inter-attempt sleep the mount and lease planes run on. A plain sleep would hold a parked or + /// stopping renewal for the whole pause; this one wakes while the workers are stopping or a park is + /// requested. It shortens a stop, not a fence loss: the fence cannot see a stop request, so a woken + /// operation still reissues unless its own liveness predicate refuses. Once the workers have + /// stopped it sleeps in full, so the farewell that follows is paced like any other write. void sleepInterruptibly(uint64_t ms); /// The mount fence's admission verdict, as `Fence::admit` expects it: may a request admitted under @@ -442,14 +466,20 @@ class CasMountRuntime RenewalDriverState active_state, RenewalDriverState returned_state, bool propagate_failure); + /// Publish a committed renewal's deadline, its start plus the lease. Requires `driver_mutex`. + void publishRenewedDeadline(uint64_t attempt_start_boot_ms); + /// One `WARNING` per pause, at the first renewal attempt after writes paused. + void warnOnceIfWritesPaused(); + /// Two attempt envelopes, saturating: what one write reserves. + uint64_t writeReservationMs() const; void renewalLoop(); void remountLoop(); ThreadFromGlobalPool makeWorker(std::function body); /// The renewal's liveness: facts the mount fence cannot see -- a shutdown request, a parked or /// park-requested driver, a pool that left `Live`. FALSE ends the renewal. bool renewalLive(bool worker_call) const; - /// Whether this node has already been asked to stop. Sampled ONCE, before the write, so a refusal - /// caused by the stop cannot be mistaken for one that preceded it. + /// Whether this node has been asked to stop; read after a refused renewal, so a stop that lands + /// mid-renewal is not mistaken for a lost lease. bool renewalCancelled() const; void tripFenceWithoutOperationalLoss(); std::unique_lock lockTerminalPublication(); @@ -457,7 +487,7 @@ class CasMountRuntime /// ---- injected environment (no `Pool` back-reference); initialized first, in this order ---- BackendPtr backend_ptr; CasRequests & mount_requests; - CasRequests & farewell_requests; + CasRequests & lease_requests; const Layout & layout; MountConfig config; String server_root_id; @@ -508,6 +538,9 @@ class CasMountRuntime uint64_t remount_handled_generation = 0; ThreadFromGlobalPool renewal_worker; ThreadFromGlobalPool remount_worker; + /// A leaf lock, never held across I/O. + mutable std::mutex renewal_progress_mutex; + RenewalProgress renewal_progress TSA_GUARDED_BY(renewal_progress_mutex); /// Counted entries into `scheduleRemount`; retained as a test-only observability seam. std::atomic schedule_remount_calls_for_test{0}; @@ -519,6 +552,7 @@ class CasMountRuntime /// Fence-generation token (rev.7 [C2]): bumped by `tripMountLost` and `armMountFence`. See /// `fenceGeneration`/`checkFenceOrThrow`. std::atomic fence_generation{0}; + std::atomic writes_paused_warned{false}; std::function arm_mount_fence_interposition_hook_for_test; /// The pool lifecycle condition (rev.7 §1). Starts `Live`. Non-terminal transitions diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.cpp index 51ff507c141f4..d05e3bfa60e2a 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPartWriteTxn.cpp @@ -10,23 +10,28 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include +#include namespace ProfileEvents { extern const Event CASBlobBodyPutAvoided; + extern const Event CASBlobPublishAttemptFailed; + extern const Event CASBlobPublishGaveUp; extern const Event CASBlobAdoptTrusted; extern const Event CASMetaPut; extern const Event CASMetaCompareSwap; extern const Event CASMetaCreateClean; - extern const Event CASMetaAdoptBackfill; + extern const Event CASMetaAbsentRepublish; extern const Event CASMetaResurrectClean; } @@ -67,6 +72,20 @@ UInt128 mintU128() return (static_cast(hi) << 64) | lo; } +/// Rate-limited, because under an outage every blob of every write fails here; the counter has them all. +/// The first failure is routine and the next attempt usually succeeds, so only a repeat is a warning. +void recordFailedPublicationAttempt( + const String & key, BlobPublicationTransport transport, int attempt, int max_attempts, const String & error) +{ + ProfileEvents::increment(ProfileEvents::CASBlobPublishAttemptFailed); + LogSeriesLimiter log(getLogger("CasBlobPublishAttempt"), /*allowed_count=*/5, /*interval_s=*/30); + const char * transport_name = transport == BlobPublicationTransport::Streaming ? "streaming" : "server-side copy"; + if (attempt == 0) + LOG_INFO(log, "Publication of CAS blob {} ({}) failed, attempt {} of {}: {}", key, transport_name, attempt + 1, max_attempts, error); + else + LOG_WARNING(log, "Publication of CAS blob {} ({}) failed, attempt {} of {}: {}", key, transport_name, attempt + 1, max_attempts, error); +} + bool isDeterministicBlobPublicationFailure(const std::exception & error) { /// A refused write, EXCEPT the class a fresh credential fixes: the engine refreshes once before it @@ -385,6 +404,14 @@ BlobUploadResult PartWriteTxn::ensureBlobPresent(const BlobUploadRequest & req) }; constexpr int max_publication_attempts = 8; + /// Every exit by exception after a failed attempt gave up on the blob, the attempt cap and the + /// loop's deadline alike. + int failed_attempts = 0; + const int exceptions_at_entry = std::uncaught_exceptions(); + SCOPE_EXIT({ + if (failed_attempts > 0 && std::uncaught_exceptions() > exceptions_at_entry) + ProfileEvents::increment(ProfileEvents::CASBlobPublishGaveUp); + }); for (int attempt = 0; attempt < max_publication_attempts; ++attempt) { requireAlive(); @@ -420,22 +447,10 @@ BlobUploadResult PartWriteTxn::ensureBlobPresent(const BlobUploadRequest & req) if (loaded) validateMetaSize(loaded->meta); - if (!loaded || loaded->meta.state == MetaState::Clean) + /// Only a `Clean` marker read after the `HEAD` permits adoption. GC deletes the marker right + /// after the body, so an absent marker may mean the body this `HEAD` saw is already gone. + if (loaded && loaded->meta.state == MetaState::Clean) { - /// Observation can produce durable metadata and dependency readiness too. - requireAdmitted("after the mandatory `HEAD`"); - if (!loaded) - { - /// A backfilled marker is a point-read hint for the next observer, so a competing - /// writer that got there first settles the same question: its outcome is not read. - ProfileEvents::increment(ProfileEvents::CASMetaAdoptBackfill); - putMetaIfAbsent( - op, - store->layout(), - ref, - BlobMeta{.state = MetaState::Clean, .condemn_round = 0, .size = logical_size}, - policy); - } requireAdmitted("before the body-put-avoided observation is recorded"); ProfileEvents::increment(ProfileEvents::CASBlobBodyPutAvoided); EventEmitter{*store}.emit([&](CasEvent & event) @@ -445,7 +460,7 @@ BlobUploadResult PartWriteTxn::ensureBlobPresent(const BlobUploadRequest & req) event.object_hash = blobIdOf(ref); event.token = present->etag.render(); event.outcome = "observed"; - event.reason = "a present non-condemned blob was observed after mandatory `HEAD`"; + event.reason = "a present blob with a `Clean` marker was observed after mandatory `HEAD`"; event.detail = {{"action", "observed"}, {"size", std::to_string(source.size)}}; }); requireAdmitted("before the observed dependency proof is returned"); @@ -454,7 +469,10 @@ BlobUploadResult PartWriteTxn::ensureBlobPresent(const BlobUploadRequest & req) BlobDepRecord{ObjectKind::Blob, BlobDependencyProof::Materialized, source.size}, BlobUploadDiagnostics{BlobMaterializationAction::Observed, std::nullopt, std::nullopt}}; } - reason = BlobPublicationReason::Condemned; + if (loaded) + reason = BlobPublicationReason::Condemned; + else + ProfileEvents::increment(ProfileEvents::CASMetaAbsentRepublish); } const bool first_publication = source.beginPublication(); @@ -495,6 +513,8 @@ BlobUploadResult PartWriteTxn::ensureBlobPresent(const BlobUploadRequest & req) /// every further request of this operation refuses the same way. if (isDeterministicBlobPublicationFailure(error) || !op.admitted()) throw; + ++failed_attempts; + recordFailedPublicationAttempt(key, transport, attempt, max_publication_attempts, error.what()); if (attempt + 1 == max_publication_attempts) throwCasWriteRetryLater(fmt::format( "PartWriteTxn::ensureBlobPresent: publication of {} remained ambiguous after {} " @@ -506,6 +526,9 @@ BlobUploadResult PartWriteTxn::ensureBlobPresent(const BlobUploadRequest & req) } catch (...) { + ++failed_attempts; + recordFailedPublicationAttempt( + key, transport, attempt, max_publication_attempts, getCurrentExceptionMessage(/*with_stacktrace=*/false)); if (attempt + 1 == max_publication_attempts) throwCasWriteRetryLater(fmt::format( "PartWriteTxn::ensureBlobPresent: publication of {} remained ambiguous after {} " @@ -526,7 +549,7 @@ BlobUploadResult PartWriteTxn::ensureBlobPresent(const BlobUploadRequest & req) event.object_hash = blobIdOf(ref); event.outcome = "published"; event.reason = reason == BlobPublicationReason::Absent - ? "blob was absent after mandatory `HEAD`" + ? "blob or its marker was absent after mandatory `HEAD`" : "blob was present but `Condemned` after mandatory `HEAD`"; event.detail = { {"action", "published"}, diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp index 794d412bf984d..e697ba441bbc3 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.cpp @@ -77,6 +77,16 @@ namespace std::atomic remount_attempt_sequence{0}; +/// What a mount that skipped the capability battery did not prove about its store. Said once per open at +/// the default level, because nothing else tells an operator the checks did not run. +void logSkippedCapabilityProbe(const String & srid, const String & pool_prefix, std::string_view reason) +{ + LOG_INFO(getLogger("CasPool"), + "CAS pool '{}' (prefix '{}'): the capability probe was skipped ({}); unverified on this mount: bucket " + "versioning, conditional-write support, list-after-write visibility", + srid, pool_prefix, reason); +} + /// Validate every writable factory's lease/request/cadence relationship before it can publish /// owner, epoch, mount, or probe authority. Decommission forces background renewal before calling /// this helper, so it is held to the same cadence window as an ordinary production mount. @@ -183,7 +193,12 @@ Pool::Pool(BackendPtr backend_, PoolConfig config_, PoolMeta meta_) config.boot_ms_fn, config.retry_sleep_fn ? config.retry_sleep_fn : mountPlaneSleepFn(), &hot_keys) - , farewell_requests(pool_backend, Fence::open(), config.boot_ms_fn, config.retry_sleep_fn, &hot_keys) + , lease_requests( + pool_backend, + Fence::open(), + config.boot_ms_fn, + config.retry_sleep_fn ? config.retry_sleep_fn : mountPlaneSleepFn(), + &hot_keys) /// The open plane's fence is the pool's teardown flag: generation 0 forever, exactly like /// `Fence::open`, but `admit` refuses once `beginTeardown` ran. A GC round, an FSCK or a probe in /// flight is then refused at its next request instead of running to completion under a disk that @@ -242,13 +257,13 @@ Pool::Pool(BackendPtr backend_, PoolConfig config_, PoolMeta meta_) [this] (const RootNamespace & ns) { cancelInflightBuildsForNamespace(ns); }, config.recovery_pre_first_request_hook_for_test) /// Mount / write-fence / build-watermark / self-remount runtime. Injected with - /// backend/layout + the mount and farewell planes + the `MountConfig` slice + `server_root_id` + the event-sink reference + the pool + /// backend/layout + the mount and lease planes + the `MountConfig` slice + `server_root_id` + the event-sink reference + the pool /// `cas_request_budget` + the `remount_attempt` callback (== `Pool::tryRemountOnce`, whose claim/ /// recovery ORCHESTRATION stays on Pool). The callback captures `this`; it is invoked only at runtime /// (post-construction). Declared/constructed AFTER `ref_ledger`, preserving the original member order /// verbatim (mount destroyed first, ledger last; both orders proven safe -- see the header note). , mount_runtime( - pool_backend, mount_requests, farewell_requests, + pool_backend, mount_requests, lease_requests, pool_layout, config.mountConfig(), config.server_root_id, event_sink_, config.cas_request_budget, [this] { return tryRemountOnce(); }) @@ -333,8 +348,8 @@ String Pool::lifecycleReasonDetail(PoolLifecycle lc) const case PoolLifecycle::TransientNotLive: return {}; case PoolLifecycle::IdentityLost: - return "identity lost — the pool sentinels are absent; access fails loud. Recover by restart or " - "SYSTEM CAS FORGET (a matching-sentinel restore does not auto-revive it)."; + return "identity lost — the pool sentinels are absent, or the owner moved to another life; access fails " + "loud. Recover by restart or SYSTEM CAS FORGET (a matching-sentinel restore does not auto-revive it)."; case PoolLifecycle::VanishedReplaced: return "data root replaced by a foreign pool (pool_id mismatch) — our generation is gone; " "restart re-registers the name."; @@ -377,6 +392,16 @@ Pool::LifecycleSnapshot Pool::lifecycleSnapshot() const snap.lifecycle = mount_runtime.lifecycle(); snap.detail = lifecycleReasonDetail(snap.lifecycle); snap.since = mount_runtime.lifecycleSinceWallS(); + if (snap.lifecycle == PoolLifecycle::Live) + { + if (const std::optional paused_at = mount_runtime.writesPausedSinceBootMs()) + { + snap.writes_paused = true; + const uint64_t now_ms = mount_runtime.bootMsNow(); + const uint64_t ago_s = (now_ms > *paused_at ? now_ms - *paused_at : 0) / 1000; + snap.since = ::time(nullptr) - static_cast(ago_s); + } + } return snap; } @@ -516,6 +541,7 @@ PoolPtr Pool::open(BackendPtr backend, PoolConfig config) /// working single-attempt client still fails closed at open instead of silently /// corrupting CAS state under blind retries later. backend->checkConditionalWriteSingleAttemptSupport(); + logSkippedCapabilityProbe(config.server_root_id, config.pool_prefix, "skip_access_check"); } } /// The catalog is mandatory for every minted pool. Make it durable before `_pool_meta`: otherwise @@ -581,13 +607,26 @@ PoolPtr Pool::open(BackendPtr backend, PoolConfig config) return store; } +MountIncarnation Pool::captureMountIncarnation() const +{ + /// Admission first, then the epoch, then the admission again: a remount publishes the new epoch + /// before it bumps the generation, so a generation that held across the epoch read dates the epoch. + const uint64_t generation = mount_requests.admit().generation(); + if (incarnation_capture_hook_for_test) + incarnation_capture_hook_for_test(); + const uint64_t writer_epoch = liveWriterEpoch(); + checkFenceOrThrow(generation); + return MountIncarnation{.writer_epoch = writer_epoch, .fence_generation = generation}; +} + void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy policy) { /// === Mount-safety startup protocol === /// STRICT ORDER: validate id → claim owner (identity) → allocate durable writer_epoch → claim - /// the mount lease (liveness) + arm the local write fence → anchor the watermark. owner / epoch - /// / mount / watermark are BOOTSTRAP-CONTROL writes: they establish the very right to write and - /// run BEFORE the write fence gates ordinary data/ref/manifest mutations. Fail closed throughout. + /// the mount lease (liveness) → re-read the owner → install the renewer + arm the local write fence + /// → anchor the watermark. owner / epoch / mount / watermark are BOOTSTRAP-CONTROL writes: they + /// establish the very right to write and run BEFORE the write fence gates ordinary data/ref/manifest + /// mutations. Fail closed throughout. /// Shared by `open` (`policy = WaitForExpiry`, `our_uuid = config.server_id`) and /// `openForDecommission` (`policy = NoWait`, `our_uuid` = the impersonated victim owner uuid; /// -- the two differ only in WHO they mount @@ -611,17 +650,23 @@ void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy pol /// can reach a slot mutation; the callback remains available for fresh conflict rechecks below. (void)observe_catalog(); - /// 2. Owner anchor — IDENTITY (clock-free). A foreign uuid fails closed; an absent owner over a - /// non-empty subtree is CORRUPTED_DATA; a fresh empty root is claimed. + /// 2. Owner anchor — IDENTITY (clock-free). A foreign uuid fails closed unless it is a verified, drained + /// retirement (rejoin); an absent owner over a non-empty subtree is CORRUPTED_DATA; a fresh empty root + /// is claimed. /// Every step of this protocol runs on the OPEN plane. These are bootstrap-control writes: they /// establish the very right to write, and the mount fence they would otherwise be gated on is /// either unarmed (a first open) or latched lost (a self-remount, which could then never reclaim). + const bool is_decommission = policy == MountClaimPolicy::NoWait; + const RetiredOwnerPolicy claim_retired_policy + = is_decommission ? RetiredOwnerPolicy::AcceptForDecommission : RetiredOwnerPolicy::RejoinIfVerified; + const RetiredOwnerPolicy recheck_retired_policy + = is_decommission ? RetiredOwnerPolicy::AcceptForDecommission : RetiredOwnerPolicy::Refuse; CasOperation owner_op = store->gc_requests.admit(); - claimOwnerOrThrow(owner_op, store->pool_layout, srid, our_uuid, observe_catalog); + const uint64_t owner_life = claimOwnerOrThrow( + owner_op, store->pool_layout, srid, our_uuid, observe_catalog, claim_retired_policy, + [s = store.get()](CasEvent e) { s->emitEvent(std::move(e)); }); + store->recorded_owner_life = owner_life; - /// Wall-clock `now_ms`, hoisted above the writer_epoch allocation below: the absent-epoch - /// branch's `DecommissionRecovery` policy needs it to judge a surviving mount's liveness before - /// the mount-lease claim (step 4) gets its own use of the same clock. const auto now_ms = []() -> uint64_t { return static_cast(std::chrono::duration_cast( @@ -635,16 +680,9 @@ void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy pol /// Mutable: a GC fence of our fresh lease during open (expiry mid-open racing a GC round) is /// recoverable — a fence costs an epoch, so the fence-recovery loop below re-allocates a fresh /// writer_epoch and re-claims (the TLA+-checked `NoPermanentWedge` invariant). - /// `epoch_policy`: `openForDecommission`'s `NoWait` gates the absent-epoch branch's mount-probe - /// on a TERMINAL (not live) surviving mount instead of authoritative absence (Phase C) — passed - /// uniformly to every call below, including the fence-recovery re-allocations, where it is inert - /// because those run with the epoch object already present. - const EpochMintPolicy epoch_policy = (policy == MountClaimPolicy::NoWait) - ? EpochMintPolicy::DecommissionRecovery - : EpochMintPolicy::NormalMount; CasOperation epoch_op = store->gc_requests.admit(); uint64_t writer_epoch = allocateWriterEpoch( - epoch_op, store->pool_layout, srid, epoch_policy, now_ms(), observe_catalog); + epoch_op, store->pool_layout, srid, observe_catalog); store->mount_runtime.setProcessEpoch(writer_epoch, std::memory_order_relaxed); /// 4. Mount lease — LIVENESS. Decide over the current mount object using the wall-clock `now_ms` @@ -715,20 +753,22 @@ void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy pol if (policy == MountClaimPolicy::WaitForExpiry) { CasOperation claim_op = store->gc_requests.admit(); + const OwnerExpectation foreign_reclaim_owner{ + .server_uuid = our_uuid, .owner_life = owner_life, .retired = RetiredOwnerPolicy::Refuse}; const bool unsafe = store->config.unsafe_remount_no_delay; if (unsafe) { - /// One bare attempt first: a slot held by OUR uuid under another epoch is reclaimed at + /// One bare attempt first: a slot held by OUR uuid under an older epoch is reclaimed at /// once under the operator's authorization, carrying the exact token this read saw so a /// slot that moves in between is refused. Every outcome but a claim (an absent slot - /// freshly minted, or a same-epoch refresh) falls through to the ordinary observed path + /// freshly minted) falls through to the ordinary observed path /// below. claim = claimMount(claim_op, store->pool_layout, srid, our_uuid, writer_epoch, now_ms(), ttl_ms, - /*proven_dead_incarnation=*/{}, emit_mount_event); + /*proven_dead_incarnation=*/{}, emit_mount_event, /*unsafe_reclaim_authorization=*/{}, foreign_reclaim_owner); if (claim.kind == MountClaimResult::LiveDoubleStart && claim.etag && claim.body && claim.body->server_uuid == our_uuid) claim = claimMount(claim_op, store->pool_layout, srid, our_uuid, writer_epoch, now_ms(), ttl_ms, - {}, emit_mount_event, /*unsafe_reclaim_authorization=*/claim.etag); + {}, emit_mount_event, /*unsafe_reclaim_authorization=*/claim.etag, foreign_reclaim_owner); } /// A `FencedSelf`, a foreign-uuid `LiveDoubleStart`/`ForeignOwner`, or a raced /// `LiveDoubleStart` the authorization above did not cover falls through here and re-runs @@ -740,13 +780,15 @@ void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy pol claim = claimMountAwaitingExpiry( claim_op, store->pool_layout, srid, our_uuid, writer_epoch, [&now_ms]() { return now_ms(); }, [raw] { return raw->bootMsNow(); }, - ttl_ms, poll_interval_ms, sleep_ms, on_wait_start, emit_mount_event); + ttl_ms, poll_interval_ms, sleep_ms, on_wait_start, emit_mount_event, foreign_reclaim_owner); } else { /// NoWait (decommission gate): a single unobserved attempt -- no bounded wait-and-retry /// for a stale-looking lease to lapse. Anything but Claimed/FencedSelf below is refused /// immediately. + /// No owner expectation: the admin impersonates the owner it retires, so whose life counts as + /// "ours" is ambiguous, and it never reclaims another uuid's body. CasOperation claim_op = store->gc_requests.admit(); claim = claimMount(claim_op, store->pool_layout, srid, our_uuid, writer_epoch, now_ms(), ttl_ms, /*proven_dead_incarnation=*/{}, emit_mount_event); @@ -761,7 +803,7 @@ void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy pol srid, max_fence_recoveries); CasOperation reallocate_op = store->gc_requests.admit(); writer_epoch = allocateWriterEpoch( - reallocate_op, store->pool_layout, srid, epoch_policy, now_ms(), observe_catalog); + reallocate_op, store->pool_layout, srid, observe_catalog); store->mount_runtime.setProcessEpoch(writer_epoch, std::memory_order_relaxed); continue; } @@ -770,9 +812,9 @@ void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy pol if (policy == MountClaimPolicy::NoWait) { /// No FORCE variant, no wait-and-observe: the decommission gate treats any live-looking - /// or foreign-owner lease as an immediate refusal. A raced write that observed nobody - /// has no holder to describe, and the lease this server merely proposed is not one -- - /// naming it would send an operator after this very process. + /// or foreign-owner lease as an immediate refusal, and names the terminal states it + /// accepts, because `expired` in system.cas_mounts is wall-clock arithmetic this claim + /// never trusts. A raced write that observed nobody has no holder to describe. const String holder = claim.body ? fmt::format("mount lease held by uuid={} epoch={} pid={} hostname={} (expires_at_ms={})", u128ToHex(claim.body->server_uuid), claim.body->writer_epoch, @@ -780,9 +822,17 @@ void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy pol : String("the conditional write that lost the mount slot observed nothing at the key, " "so the holder is unknown to this server"); throw Exception(ErrorCodes::ABORTED, - "CAS decommission '{}': pool member is alive or contended — {}. Refusing (no FORCE " - "variant exists; stop the server or wait for its lease to lapse).", - srid, holder); + "CAS decommission '{}': pool member is alive or contended — {}. Refusing; there is no FORCE " + "variant. A member can be removed only once its mount slot is terminal in system.cas_mounts " + "(server_root_id = '{}'): state = 'terminated' (the server stopped cleanly) or state = 'fenced' " + "(GC fenced its lease). A hard-killed member shows state = 'expired' first and is not removable " + "yet. The disk's GC leader fences it at the start of a GC round, once it has seen the lease unchanged " + "for about one cas_mount_lease_ttl_ms plus one cas_mount_renew_period_ms, so the wait can be one " + "GC round longer. SYSTEM CAS GC RUN on the leader (is_leader = 1 in system.cas_mounts) can " + "trigger the fence, but the run that fences must come at least that long after the first run " + "that saw the lease. " + "Then rerun this command.", + srid, holder, srid); } /// LiveDoubleStart (waited out the bound → a live twin) or ForeignOwner → fail closed /// with the actionable, multi-line startup error. @@ -790,6 +840,13 @@ void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy pol } claimed_prior = claim.prior; + /// The owner may have moved since the claim began: a tombstone, a rollback or a rejoin. Nothing + /// below runs under an owner this process did not start under. + CasOperation recheck_op = store->gc_requests.admit(); + recheckOwnerAfterClaim(recheck_op, store->pool_layout, srid, + OwnerExpectation{.server_uuid = our_uuid, .owner_life = owner_life, .retired = recheck_retired_policy}, + writer_epoch, emit_mount_event); + /// The mount object now holds OUR live (uuid, epoch) body. `installRenewer` constructs the renewer /// -- which ADOPTS that very (uuid, epoch) slot rather than self-tripping the double-start guard -- /// AND wires its `minActive` build-watermark reader, its event sink, and the fence-coupling @@ -808,7 +865,7 @@ void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy pol store->mount_runtime.renewerReset(); CasOperation refenced_epoch_op = store->gc_requests.admit(); writer_epoch = allocateWriterEpoch( - refenced_epoch_op, store->pool_layout, srid, epoch_policy, now_ms(), observe_catalog); + refenced_epoch_op, store->pool_layout, srid, observe_catalog); store->mount_runtime.setProcessEpoch(writer_epoch, std::memory_order_relaxed); continue; } @@ -820,7 +877,7 @@ void Pool::mountWritable(PoolPtr & store, UInt128 our_uuid, MountClaimPolicy pol /// the three `MountPriorState`s with no such proof (`UncleanUnsafe` has no proof at all, not merely /// no proof of a CLEAN death: it is the operator's explicit `cas_unsafe_remount_no_delay` /// acceptance of that risk, with no observation behind it). `Clean`, drained farewell, and `None`, a - /// fresh mount / same-epoch refresh with nothing to hand over, are the proven ones. + /// fresh mount with nothing to hand over, are the proven ones. /// An EXHAUSTIVE switch, not a positive allowlist -- a future `MountPriorState` /// enumerator with no proof of clean death must fail the BUILD (a missing `-Wswitch` case), never /// silently fall through to "clean". @@ -931,9 +988,14 @@ PoolPtr Pool::openForDecommission(BackendPtr backend, PoolConfig config, const S /// refusing an ordinary mount costs availability and protects data, whereas refusing a /// decommission strands a pool with a dead replica in it and leaves the operator no way forward. config.skip_access_check = true; + logSkippedCapabilityProbe( + victim_srid, config.pool_prefix, fmt::format("decommission of pool member '{}'", victim_srid)); /// The admin claim must be RENEWED like any writable mount: the host disk may be observe-only /// (background_watermark=false), but an unrenewed claim (TTL ~30s) aborts any long drain midway. config.background_watermark = true; + /// Bounded by the lease, like the startup re-anchor: an admin claim that cannot renew ends rather than + /// pauses. + config.bounded_background_renewal = true; validateWritableMountTiming(config); Layout layout(config.pool_prefix); @@ -942,7 +1004,7 @@ PoolPtr Pool::openForDecommission(BackendPtr backend, PoolConfig config, const S /// below is then EXACTLY the crash-recovery reclaim semantics (`MountClaimPolicy::NoWait`): a /// fenced/terminated/clean-farewell lease reclaims; a live lease refuses immediately (no bounded /// observation wait -- see `mountWritable`). Owner anchor absent + mount absent = nothing to - /// decommission. + /// decommission. A tombstoned owner of the victim is accepted: a rerun resumes over its own tombstone. /// The open plane: this factory impersonates the victim to take its mount, so there is no lease of /// ours to be gated on until the claim below establishes one. `config.retry_sleep_fn` must travel /// with `config.boot_ms_fn`: a retry loop bound to a frozen test clock that only a fake sleep @@ -963,6 +1025,15 @@ PoolPtr Pool::openForDecommission(BackendPtr backend, PoolConfig config, const S } config.server_id = *victim_uuid; + /// A member of this build always has a counter next to its owner. An absent counter is a state left by + /// 26.6.4 or by a process that died between the owner create and the counter create; minting one here + /// would hand out epoch 1 again over whatever that member wrote. + if (!owner_op.read(layout.epochKey(victim_srid), Retry::standard())) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS decommission '{}': no epoch counter '{}'; a decommission never creates one. Start the server with a new " + "server_root_id; see decommission compatibility in the CAS migration docs", + victim_srid, layout.epochKey(victim_srid)); + backend->checkConditionalWriteSingleAttemptSupport(); /// Decommission operates on an EXISTING pool member (an owner anchor / mount lease was just found), so /// `_pool_meta` must already be present. It never bootstraps: `allow_mint=false` so an absent meta @@ -978,6 +1049,7 @@ PoolPtr Pool::openForDecommission(BackendPtr backend, PoolConfig config, const S /// Private ctor: make_shared cannot reach it. PoolPtr store(new Pool(std::move(backend), std::move(config), std::move(meta))); store->setEventSink(std::move(store->config.event_sink)); + store->decommission_admin = true; /// Register-before-first-write, belt-and-braces: same invariant `open` asserts. chassert(store->isAlgoAdmitted(write_algo)); @@ -1041,7 +1113,7 @@ Pool::~Pool() config.teardown_phase2_throw_for_test(); const bool ref_lanes_drained = ref_ledger.drainRefLanesForShutdown( config.cas_request_budget.attemptEnvelopeMs() + config.cas_request_budget.lease_safety_margin_ms); - drained = ref_lanes_drained && !writerCleanupDutiesPending(); + drained = ref_lanes_drained && !writerCleanupDutiesPending() && !farewell_withheld; }, "CAS pool teardown: draining ref lanes"); /// 3. Retire the merged heartbeat. This always runs after phase 2, even when that phase failed, and @@ -1408,7 +1480,7 @@ bool Pool::tryRemountOnce() /// fail-loud TERMINAL state — the remount worker self-exits at its next boundary (see /// `CasMountRuntime::remountTerminal`), so there is no demoted observer. if (mount_runtime.lifecycle() != PoolLifecycle::IdentityLost) - mount_runtime.enterIdentityLost(); + mount_runtime.enterIdentityLost(gate.reason); return false; case LifecycleGateVerdict::StayTransient: /// Uncertain — remain transient and let the recovery loop retry. The probe's own reason is @@ -1444,16 +1516,36 @@ bool Pool::tryRemountOnce() /// Require the catalog before allocating a new epoch or replacing the mount slot. step = "ref_catalog_observe"; (void)observe_catalog(); - step = "owner_claim"; /// The open plane throughout: the mount fence is latched lost here (starting the renewer does /// not clear it), so an operation admitted under the fence could never make the claim that /// re-establishes it. CasOperation owner_op = gc_requests.admit(); - claimOwnerOrThrow(owner_op, pool_layout, srid, our_uuid, observe_catalog); + /// No retry can undo a foreign, retired or rewritten owner, so stop for good before the claim and the epoch. + step = "owner_check"; + error = remountOwnerRefusal(owner_op, pool_layout, srid, our_uuid, recorded_owner_life); + if (!error.empty()) + { + if (decommission_admin) + error = fmt::format("the SYSTEM CAS DROP POOL MEMBER run for '{}' lost its admin claim and does not " + "remount; the run reports its own outcome ({})", srid, error); + mount_runtime.enterIdentityLost(error); + return false; + } + step = "owner_claim"; + const uint64_t owner_life = claimOwnerOrThrow(owner_op, pool_layout, srid, our_uuid, observe_catalog); + if (owner_life != recorded_owner_life) + { + /// A rejoin or a rollback moved the owner since this process started. No retry can bring the old + /// life back, so stop for good before an epoch is allocated; a restart runs under the current life. + error = fmt::format("the owner moved from life {} to life {} since this process started; restart the server", + recorded_owner_life, owner_life); + mount_runtime.enterIdentityLost(error); + return false; + } step = "writer_epoch_allocate"; CasOperation epoch_op = gc_requests.admit(); const uint64_t writer_epoch = allocateWriterEpoch( - epoch_op, pool_layout, srid, EpochMintPolicy::NormalMount, 0, observe_catalog); + epoch_op, pool_layout, srid, observe_catalog); result_writer_epoch = writer_epoch; /// Mount-slot writer audit: `this` is already fully open (setEventSink ran long ago), so @@ -1477,16 +1569,23 @@ bool Pool::tryRemountOnce() "epoch={} expires_at_ms={}) for up to ~{} ms before reclaiming", srid, u128ToHex(held.server_uuid), held.writer_epoch, held.expires_at_ms, threshold_ms); }, - emit_mount_event); + emit_mount_event, + OwnerExpectation{.server_uuid = our_uuid, .owner_life = recorded_owner_life, .retired = RetiredOwnerPolicy::Refuse}); if (claim.kind != MountClaimResult::Claimed) { LOG_WARNING(getLogger("CasPool"), "CAS self-remount '{}': mount not claimable ({}); will retry", srid, - claim.kind == MountClaimResult::ForeignOwner ? "foreign owner — never taking over" + claim.kind == MountClaimResult::ForeignOwner ? "foreign live body, or the owner did not match — not taking over" : "a live twin holds the lease"); return false; } + step = "owner_recheck"; + CasOperation recheck_op = gc_requests.admit(); + recheckOwnerAfterClaim(recheck_op, pool_layout, srid, + OwnerExpectation{.server_uuid = our_uuid, .owner_life = recorded_owner_life, .retired = RetiredOwnerPolicy::Refuse}, + writer_epoch, emit_mount_event); + /// NO WAIT HERE, AND THE ASSERT IS WHAT REPLACES IT. This used to pay `materialization_grace_ms` /// whenever this incarnation's own ref lanes had not provably settled before the fence tripped: /// an unresolved (still-wedged) ref-log conditional `PUT` from the dying epoch could otherwise @@ -1964,19 +2063,19 @@ void Pool::setCasRetrySleepForTest(std::function sleep_fn) { /// All three planes, not just the ledger's: a test that replaces the retry sleep must not be left /// with a real one on the plane the site under test happens to use. - farewell_requests.setSleepFnForTest(sleep_fn); ref_ledger.setCasRetrySleepForTest(sleep_fn); - /// `CasRequests` falls back to the engine's plain sleep for an empty argument -- which is neither - /// the mount plane's nor the open plane's default. Re-install both, so clearing the seam cannot - /// leave a parked renewal held for a whole capped backoff, or the open plane deaf to a teardown. + /// `CasRequests` falls back to the engine's plain sleep for an empty argument -- which is none of + /// the planes' defaults. Re-install them, so clearing the seam cannot leave a parked renewal held + /// for a whole pause, or the open plane deaf to a teardown. gc_requests.setSleepFnForTest(sleep_fn ? sleep_fn : openPlaneSleepFn()); + lease_requests.setSleepFnForTest(sleep_fn ? sleep_fn : mountPlaneSleepFn()); mount_requests.setSleepFnForTest(sleep_fn ? std::move(sleep_fn) : mountPlaneSleepFn()); } void Pool::setCasRequestNowFnForTest(std::function now_fn) { mount_requests.setNowFnForTest(now_fn); - farewell_requests.setNowFnForTest(now_fn); + lease_requests.setNowFnForTest(now_fn); gc_requests.setNowFnForTest(std::move(now_fn)); } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h index 0f565d93f3de6..9dd2754020dc6 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasPool.h @@ -184,6 +184,8 @@ struct PoolConfig uint64_t gc_io_concurrency = 16; /// Tests drive `renewWatermarkOnce` explicitly; gates both persistent runtime workers. bool background_watermark = false; + /// See `MountConfig::bounded_background_renewal`. + bool bounded_background_renewal = false; /// Installed on the pool before a writable mount can start its runtime-owned workers. CasEventSink event_sink = {}; @@ -256,10 +258,10 @@ struct PoolConfig /// boot clock (`Pool::bootMs`); injected by tests to drive the fence deadline deterministically. std::function boot_ms_fn = {}; - /// The inter-attempt sleep for the mount, farewell and GC request planes (`mount_requests`, - /// `farewell_requests`, `gc_requests`), installed at their CONSTRUCTION -- before this `Pool` has - /// claimed or read anything. Empty = each plane's own production default (an interruptible real - /// sleep for the mount and GC planes, `CasRequests`'s own real sleep for the farewell plane). A test + /// The inter-attempt sleep for the mount, lease and GC request planes (`mount_requests`, + /// `lease_requests`, `gc_requests`), installed at their CONSTRUCTION -- before this `Pool` has + /// claimed or read anything. Empty = each plane's own production default, an interruptible real + /// sleep. A test /// that also freezes `boot_ms_fn` must supply a matching sleep here: a retry loop bound to a clock /// that only moves when this function is called would otherwise retry forever against a REAL sleep /// that never calls it, because the deadline it measures against never appears to elapse. @@ -355,10 +357,17 @@ struct PoolConfig .boot_ms_fn = boot_ms_fn, .wait_sleep_fn = wait_sleep_fn, .worker_factory = worker_factory, + .bounded_background_renewal = bounded_background_renewal, }; } }; +/// The live incarnation at one instant: the writer epoch and the fence generation it was admitted under. +struct MountIncarnation +{ + uint64_t writer_epoch = 0; + uint64_t fence_generation = 0; +}; struct PartWriteInfo { @@ -421,19 +430,21 @@ class Pool : public std::enable_shared_from_this /// Construct a pool after backend capabilities and durable pool metadata have been validated. /// For a writable configuration, `open` also claims the configured mount before returning; a /// read-only configuration returns an instance that performs no mutating startup operations. + /// A start over a verified, drained retirement of its `server_root_id` rejoins it as an empty member + /// (`claimOwnerOrThrow`). static PoolPtr open(BackendPtr backend, PoolConfig config); - /// Admin writer mount of the VICTIM `server_root_id`, for `SYSTEM CAS DROP POOL - /// MEMBER`. Impersonates the victim's - /// owner uuid (`readOwnerUuid`, or -- when the owner anchor itself is missing -- recovered from a - /// lingering mount lease) and mounts writable under `MountClaimPolicy::NoWait`: a live victim - /// lease is an immediate `ABORTED` refusal (no wait-and-observe, no FORCE variant), unlike the - /// bounded reclaim wait a normal `open` pays. Throws `BAD_ARGUMENTS` when there is nothing to - /// decommission (no owner anchor and no mount lease for `victim_srid`). + /// Admin writer mount of the VICTIM `server_root_id`, for `SYSTEM CAS DROP POOL MEMBER`. + /// Impersonates the victim's owner uuid (`readOwnerUuid`, or, when the owner was deleted by hand, the uuid + /// of the surviving lease) and mounts writable under `MountClaimPolicy::NoWait`: a live victim lease is an + /// immediate `ABORTED` refusal; a tombstoned owner of the victim is accepted; a member without an epoch + /// counter is refused (`CORRUPTED_DATA`). Throws `BAD_ARGUMENTS` when neither owner nor lease exists. static PoolPtr openForDecommission(BackendPtr backend, PoolConfig config, const String & victim_srid); /// Stop the mount-renewal and remount activity, drain ref-log work, and release the owned /// backend-facing components in their dependency order. Destruction is also the clean-farewell /// path for a writable mount, so it must complete before the owning backend is released. ~Pool(); + /// The teardown writes no clean farewell: a successor then waits for this lease to be fenced or observed dead. + void withholdFarewell() noexcept { farewell_withheld = true; } bool tryDispatchDetached(std::function task); /// Marks this pool as being torn down. The open request plane refuses every further admission and @@ -511,6 +522,9 @@ class Pool : public std::enable_shared_from_this /// Throws the typed transient refusal (`throwCasTransientUnavailable`) unless the fence is currently /// held AND still at `admitted_generation`. void checkFenceOrThrow(uint64_t admitted_generation) const { mount_runtime.checkFenceOrThrow(admitted_generation); } + /// Throws the transient refusal when the fence moved between the two reads, so the pair is never mixed. + MountIncarnation captureMountIncarnation() const; + void setIncarnationCaptureHookForTest(std::function hook) { incarnation_capture_hook_for_test = std::move(hook); } /// ---- pool lifecycle condition (rev.7 §1; owned by `mount_runtime`) ---- /// Atomic read of the current lifecycle condition. Thin forwarder; safe to call from any thread. @@ -543,8 +557,11 @@ class Pool : public std::enable_shared_from_this PoolLifecycle lifecycle = PoolLifecycle::Live; String detail; time_t since = 0; + /// `Live`, but the lease has too little time left to admit a write; `since` is then when writes paused. + bool writes_paused = false; }; LifecycleSnapshot lifecycleSnapshot() const; + RenewalProgress renewalProgress() const { return mount_runtime.renewalProgress(); } /// `SYSTEM CAS FORGET` — the operator force-Vanish (spec §5). Drives THIS pool to /// `Vanished(forgotten)` with the fence-first protocol, node-locally, regardless of the current @@ -760,20 +777,18 @@ class Pool : public std::enable_shared_from_this /// operation admitted here is refused the moment the fence trips, is re-armed under a fresh lease /// incarnation, or runs out of room before the lease expires. CasRequests & mountRequests() { return mount_requests; } - /// The farewell plane, on an open fence -- shared with the mount-lease renewer's own claim/adopt, - /// not only its release: a self-remount claims with the fence already latched lost, so gating the - /// claim on the fence could never reclaim, and refusing the farewell because the mount fence has - /// already run down would leave the slot looking live until GC fences it out. - CasRequests & farewellRequests() { return farewell_requests; } + /// The lease plane, on an open fence: the renewer's claim and adoption, the remount re-anchor and + /// the farewell. None of them may be refused for want of the lease they establish or hand back. + CasRequests & leaseRequests() { return lease_requests; } /// The open-fence plane: GC, the offline tools, this pool's own reads, and the bootstrap-control /// claims. None of them hold a mount lease -- the claims are what ESTABLISHES one, so gating them /// on the fence would make a self-remount, which runs with the fence latched lost, unable ever to /// reclaim. CasRequests & openRequests() { return gc_requests; } - /// The owning `BackendPtr` itself (not just a reference into it): the decommission slot-retirement - /// decommission step (`CasDecommission.cpp`) must keep the backend alive across `admin.reset()` -- the graceful - /// close that stamps the mount's farewell -- to physically delete the control objects afterward. A - /// bare `Backend &` from `backend()` would dangle the instant the owning `Pool` is destroyed. + /// The owning `BackendPtr` itself (not just a reference into it): decommission (`CasDecommission.cpp`) + /// keeps the backend alive across `admin.reset()` -- the graceful close that stamps the mount's farewell -- + /// to read the farewell back and mark the retirement verified. A bare `Backend &` from `backend()` would + /// dangle the instant the owning `Pool` is destroyed. BackendPtr poolBackendPtr() const { return pool_backend; } /// Staging write surface for `PartWriteTxn`: thin delegate onto the ref ledger, so a staging write @@ -1173,9 +1188,9 @@ class Pool : public std::enable_shared_from_this return std::forward(mutation)(); } - /// The mount plane's inter-attempt sleep: interruptible, so a parked or stopping renewal is not - /// held for a whole capped backoff. Named rather than inlined because the test seam has to be able - /// to put it back. + /// The mount and lease planes' inter-attempt sleep: interruptible, so a parked or stopping renewal is + /// not held for a whole pause. Named rather than inlined because the test seam has to be able to put + /// it back. std::function mountPlaneSleepFn() { return [this](uint64_t ms) { mount_runtime.sleepInterruptibly(ms); }; @@ -1208,7 +1223,8 @@ class Pool : public std::enable_shared_from_this /// config they take their clock from. `mutable` because issuing a request is not a change to the /// pool: a `const` observer still has to read the store. mutable CasRequests mount_requests; - mutable CasRequests farewell_requests; + std::function incarnation_capture_hook_for_test; + mutable CasRequests lease_requests; mutable CasRequests gc_requests; std::shared_ptr detached_work = std::make_shared(); @@ -1219,6 +1235,7 @@ class Pool : public std::enable_shared_from_this /// Sticky fail-close bit for the destructor's allocation-failure path. If a duty could not enter /// the queue, no mount teardown may claim a clean farewell even though the guarded map is empty. std::atomic writer_cleanup_queue_failed{false}; + std::atomic farewell_withheld{false}; /// CAS mixed-algo pools: monotone in-memory cache of `algos_used`, seeded from /// `meta.algos_used` at open. Guards `isAlgoAdmitted`/`refreshAdmittedAlgos` -- ITS OWN mutex, @@ -1276,6 +1293,13 @@ class Pool : public std::enable_shared_from_this /// the Pool-side orchestration, so it stays on Pool. std::mutex remount_mutex; + /// Set once by the owner claim in `mountWritable`; a remount checks against it and never updates it. + uint64_t recorded_owner_life = 0; + + /// The pool impersonates a victim for `SYSTEM CAS DROP POOL MEMBER` (`openForDecommission`). Its own tombstone + /// ends its remount like any retired owner, but the reason names the run, not a decommissioned server. + bool decommission_admin = false; + /// The single home of the [D5] per-lifecycle reason detail (spec §1) — the human-readable text that /// names the ACTUAL sub-state, WITHOUT the `content-addressed pool '' ` prefix. Both /// `throwIfLifecycleTerminal` (which prefixes it and throws) and the non-gated `lifecycleSnapshot` diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.cpp index d7a30304eb688..0ad8dee9b5d7d 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefCkpt.cpp @@ -267,19 +267,13 @@ CkptPublishOutcome publishCkpt(CasOperation & op, const Layout & layout, const N /// get there, so the write is resolved rather than skipped. return decisions > 1 ? CkptPublishOutcome::Published : CkptPublishOutcome::IdenticalSkip; } - if (const auto * gave_up = std::get_if(&result)) - { - /// A lost fence is an expected, transient control signal, so it is a value rather than a - /// throw. It says nothing about the object: the engine reports it both before an attempt is - /// sent and after one is proven durable, so the caller must re-read rather than assume. - /// Every other give-up leaves the contribution unpublished, and the caller must be told that - /// rather than left to assume it landed. - if (gave_up->why == GaveUp::Why::FenceLost) - return CkptPublishOutcome::FencedOut; - throwCasWriteRetryLater("CAS _ckpt for namespace '" + life.ns.string() - + "': persistent CAS contention, the checkpoint contribution was not published"); - } - /// Only `Conflict` and `Refused` are left, and both are an exception. + /// A lost fence is an expected, transient control signal, so it is a value rather than a throw. + /// It says nothing about the object: the engine reports it both before an attempt is sent and + /// after one is proven durable, so the caller must re-read rather than assume. + if (const auto * gave_up = std::get_if(&result); gave_up && gave_up->why == GaveUp::Why::FenceLost) + return CkptPublishOutcome::FencedOut; + /// Every other give-up, conflict and refusal leaves the contribution unpublished, and `orThrow` + /// tells the caller so with the real reason. const String what = "CAS _ckpt for namespace '" + life.ns.string() + "'"; orThrow(std::move(result), what); UNREACHABLE(); diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.cpp index 71e6f6c045129..7ee545d750724 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasRefLedger.cpp @@ -86,6 +86,13 @@ namespace DB::Cas namespace { +/// Closing clauses for a write that was not attempted because an earlier transaction of the same +/// table is unresolved. Each carries the outcome phrase the earlier transaction's own writer saw. +constexpr std::string_view outcome_unknown_tail = "it may or may not take effect, and retrying the earlier write may duplicate it; this write was not attempted"; +constexpr std::string_view durable_unconfirmed_tail + = "the next ref-log recovery installs it, so it may still take effect, and retrying the earlier write " + "may duplicate it; this write was not attempted"; + /// The relink confirm's logger, resolved once. `LOG_IMPL` evaluates its logger argument BEFORE testing /// the level, so `getLogger` at the call site would take the global logger-registry lock on every /// refusal even with tracing off -- and `confirmExactRef` refuses while holding pool-wide append @@ -262,7 +269,8 @@ void CasRefLedger::refuseUnlessAdmitted(const CasOperation & op, std::string_vie throwCasTransientUnavailable( fmt::format("content-addressed pool '{}'", server_root_id), fmt::format("{}: the operation is no longer admitted -- the mount lease has too little time left " - "for another request, or this table's runtime was detached", what)); + "for another request (writes are paused and resume once the lease renewal is confirmed), " + "or this table's runtime was detached", what)); } void CasRefLedger::setCasRetrySleepForTest(std::function sleep_fn) @@ -2300,16 +2308,16 @@ CasRefLedger::resolveWedgeOnce(const RootNamespace & ns, const std::shared_ptr(*invalid_lane_state))); } return result; @@ -2357,9 +2365,11 @@ CasRefLedger::resolveWedgeOnce(const RootNamespace & ns, const std::shared_ptrmessage)); + "refused ({}), which says nothing about the earlier ambiguous attempt — the lane stays wedged; " + "wedged txn {}-{} outcome is unknown: {}", + ns.string(), wedge.txn_id.writer_epoch, wedge.txn_id.ref_sequence, refused->message, + wedge.txn_id.writer_epoch, wedge.txn_id.ref_sequence, outcome_unknown_tail)); return result; } @@ -2428,7 +2440,11 @@ CasRefLedger::resolveWedgeOnce(const RootNamespace & ns, const std::shared_ptrstate_mutex); if (same_wedge_under_lock()) requireRecovery(*rt, ns, "wedged-frontier publication"); @@ -2438,9 +2454,10 @@ CasRefLedger::resolveWedgeOnce(const RootNamespace & ns, const std::shared_ptrstate_mutex); if (same_wedge_under_lock()) requireRecovery(*rt, ns, "wedged-frontier publication fence"); @@ -2631,7 +2648,8 @@ CasRefLedger::resolveWedgeOnce(const RootNamespace & ns, const std::shared_ptrappend_attempt; + /// Callers must learn whether a retry can duplicate this transaction, so the two outcomes that + /// leave that open say so, and keep the original code and cause. + struct DurableUnconfirmed + { + int code; + String cause; + }; + /// A precommit binding is never published as a part. What a durable promotion means to the table + /// depends on the engine (`MergeTree` activates it at restart, `ReplicatedMergeTree` detaches it as + /// unexpected), which this layer cannot see, so the text claims only that the write may take effect. + /// The chunk can mix kinds, so every waiter gets the text of its own step. + const auto fail_durable_unconfirmed = [&](const DurableUnconfirmed & failure) + { + for (const auto & item : chunk_survivors) + { + const std::vector> one{item}; + if (item->kind == RootMutationKind::Precommit) + { + complete_error(one, std::make_exception_ptr(Exception(failure.code, + "CAS ref-log txn {}-{} for namespace '{}' is durable but was not acknowledged ({}); the " + "next ref-log recovery installs the precommit binding, but the part is never published, " + "so its rows will not appear", + id.writer_epoch, id.ref_sequence, ns.string(), failure.cause))); + } + else + { + complete_error(one, std::make_exception_ptr(Exception(failure.code, + "CAS ref-log txn {}-{} for namespace '{}' is durable but was not acknowledged ({}); the " + "next ref-log recovery installs it, so the write may still take effect and a retry may " + "duplicate it", + id.writer_epoch, id.ref_sequence, ns.string(), failure.cause))); + } + } + }; + const auto outcome_unknown = [&](int code, const String & cause) + { + return std::make_exception_ptr(Exception(code, + "CAS ref-log txn {}-{} for namespace '{}' outcome is unknown ({}): it may or may not take " + "effect; a retry may duplicate it", + id.writer_epoch, id.ref_sequence, ns.string(), cause)); + }; + std::optional written; try { @@ -3740,14 +3805,13 @@ bool CasRefLedger::commitRefChunk(const RootNamespace & ns, const std::shared_pt /// local failure raised at a point where an attempt may already have been sent. That is /// ambiguous, and it therefore transfers ownership from `Writing` to `Wedged`; the exact /// attempt remains installed. - const std::exception_ptr write_error = std::current_exception(); { std::lock_guard lock(rt->state_mutex); if (rt->lane_state == RefLaneState::Writing && rt->append_attempt && rt->append_attempt->txn_id == id) rt->lane_state = RefLaneState::Wedged; } - complete_error(chunk_survivors, write_error); + complete_error(chunk_survivors, outcome_unknown(getCurrentExceptionCode(), getCurrentExceptionMessage(false, false, false, false))); return false; } @@ -3786,11 +3850,10 @@ bool CasRefLedger::commitRefChunk(const RootNamespace & ns, const std::shared_pt rt->lane_state = RefLaneState::Wedged; } ProfileEvents::increment(ProfileEvents::CASRefAppendWedged); - complete_error(chunk_survivors, makeCasWriteRetryLaterExceptionPtr(fmt::format( - "CAS ref-log append for namespace '{}': the store refused txn {}-{}'s create and the " - "settling read named no occupant (observed {}) — the append lane is wedged until the " - "SAME key resolves durable or a conclusive rejection is observed", - ns.string(), id.writer_epoch, id.ref_sequence, detail::renderObservation(conflict->seen)))); + complete_error(chunk_survivors, outcome_unknown(ErrorCodes::NETWORK_ERROR, fmt::format( + "the store refused the create and the settling read named no occupant (observed {}); " + "the append lane is wedged until the same key resolves", + detail::renderObservation(conflict->seen)))); return false; } @@ -3903,7 +3966,8 @@ bool CasRefLedger::commitRefChunk(const RootNamespace & ns, const std::shared_pt ProfileEvents::increment(ProfileEvents::CASRefAppendDefiniteFailure); complete_error(chunk_survivors, makeCasWriteRetryLaterExceptionPtr(fmt::format( "CAS ref-log append for namespace '{}' definitively failed ({}); " - "cached state is unchanged and txn id {}-{} was never used (a retry re-derives it)", + "cached state is unchanged and txn id {}-{} was never used (a retry re-derives it); " + "nothing is durable, so a retry cannot duplicate this transaction", ns.string(), refused->message, id.writer_epoch, id.ref_sequence))); return false; } @@ -3954,7 +4018,8 @@ bool CasRefLedger::commitRefChunk(const RootNamespace & ns, const std::shared_pt complete_error(chunk_survivors, makeCasWriteRetryLaterExceptionPtr(fmt::format( "CAS ref-log append for namespace '{}' txn {}-{} was refused BEFORE any request was " "sent — the append lane is NOT wedged (nothing can be durable, so there is " - "nothing to resolve) and the txn id is not consumed (a retry re-derives it)", + "nothing to resolve) and the txn id is not consumed (a retry re-derives it); " + "nothing is durable, so a retry cannot duplicate this transaction", ns.string(), id.writer_epoch, id.ref_sequence))); return false; } @@ -3965,11 +4030,9 @@ bool CasRefLedger::commitRefChunk(const RootNamespace & ns, const std::shared_pt rt->lane_state = RefLaneState::Wedged; } ProfileEvents::increment(ProfileEvents::CASRefAppendWedged); - complete_error(chunk_survivors, makeCasWriteRetryLaterExceptionPtr(fmt::format( - "CAS ref-log append for namespace '{}' txn {}-{} is UNCERTAIN (last observed {}) — " - "the append lane is wedged until the SAME key resolves durable or a conclusive rejection " - "is observed; this outcome is unproven, not failure", - ns.string(), id.writer_epoch, id.ref_sequence, detail::renderObservation(gave_up->last_seen)))); + complete_error(chunk_survivors, outcome_unknown(ErrorCodes::NETWORK_ERROR, fmt::format( + "gave up after a request was sent, last observed {}; the append lane is wedged until the " + "same key resolves", detail::renderObservation(gave_up->last_seen)))); return false; } @@ -3989,12 +4052,12 @@ bool CasRefLedger::commitRefChunk(const RootNamespace & ns, const std::shared_pt } catch (...) { - const std::exception_ptr frontier_error = std::current_exception(); { std::lock_guard lock(rt->state_mutex); requireRecovery(*rt, ns, "committed-frontier publication"); } - complete_error(chunk_survivors, frontier_error); + fail_durable_unconfirmed({getCurrentExceptionCode(), + "publishing its checkpoint frontier was not confirmed: " + getCurrentExceptionMessage(false, false, false, false)}); return false; } if (frontier_outcome == CkptPublishOutcome::FencedOut) @@ -4003,17 +4066,15 @@ bool CasRefLedger::commitRefChunk(const RootNamespace & ns, const std::shared_pt std::lock_guard lock(rt->state_mutex); requireRecovery(*rt, ns, "committed-frontier publication fence"); } - complete_error(chunk_survivors, makeCasWriteRetryLaterExceptionPtr(fmt::format( - "CAS ref-log append for namespace '{}': txn {}-{} is durable, but the mount fence " - "moved before its checkpoint frontier was published; the lane needs recovery", - ns.string(), id.writer_epoch, id.ref_sequence))); + fail_durable_unconfirmed({ErrorCodes::NETWORK_ERROR, + "the mount fence moved before the frontier was published"}); return false; } if (carve_hook_for_test) carve_hook_for_test(CarvePhaseForTest::PostDurableInstall); bool install_refused = false; - std::exception_ptr install_admission_error; + std::optional install_admission_error; { std::lock_guard lock(rt->state_mutex); /// The checkpoint CAS can succeed and the fence can move before the state lock is @@ -4026,8 +4087,10 @@ bool CasRefLedger::commitRefChunk(const RootNamespace & ns, const std::shared_pt } catch (...) { - install_admission_error = std::current_exception(); requireRecovery(*rt, ns, "post-frontier install admission"); + install_admission_error = DurableUnconfirmed{getCurrentExceptionCode(), + "its install was not admitted after the frontier was published: " + + getCurrentExceptionMessage(false, false, false, false)}; } /// Only this leader mutates `rt->state`, so the candidate's base snapshot is still the /// current one: there is one append-lane leader per table at a time (the `leader_active` @@ -4097,15 +4160,16 @@ bool CasRefLedger::commitRefChunk(const RootNamespace & ns, const std::shared_pt } if (install_admission_error) { - complete_error(chunk_survivors, install_admission_error); + fail_durable_unconfirmed(*install_admission_error); return false; } if (install_refused) { complete_error(chunk_survivors, std::make_exception_ptr(Exception( ErrorCodes::LOGICAL_ERROR, - "CAS ref-log append for namespace '{}': txn {}-{} is durable but this table changed " - "before installation; the lane needs recovery and refuses later writes until replay", + "CAS ref-log append for namespace '{}': txn {}-{} is durable but was not acknowledged " + "(this table changed before installation); the lane needs recovery and refuses later " + "writes until replay", ns.string(), id.writer_epoch, id.ref_sequence))); return false; } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.cpp index d50a9fb3f85b6..e8a9c1d2e1b15 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.cpp @@ -9,8 +9,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -294,7 +296,7 @@ void emitMountRenewEvent( event.type = CasEventType::WatermarkRenew; event.outcome = String{outcome}; event.reason = outcome == "recovered" - ? "CAS mount renewal recovered before its confirmed lease-safety deadline" + ? "CAS mount renewal committed after a retry or a settling read" : "CAS mount renewal ended without retained authority and fenced the mount"; event.detail = { {"server_root_id", *context.server_root_id}, @@ -381,12 +383,13 @@ void deliverMountRenewObservability( LOG_INFO( getLogger("CasMountLeaseRenewer"), "CAS mount renewal '{}' recovered after {} physical attempts in {} ms " - "(classification={}, confirmed_deadline_boot_ms={})", + "(classification={}, confirmed_deadline_boot_ms={}, remaining_confirmed_budget_ms={})", *context.server_root_id, context.attempts_sent, elapsedSince(context.observability_start_boot_ms, now_boot_ms), classification, - context.confirmed_deadline_boot_ms); + context.confirmed_deadline_boot_ms, + remainingConfirmedBudget(context, now_boot_ms)); } catch (...) { @@ -408,12 +411,13 @@ void deliverMountRenewObservability( LOG_WARNING( getLogger("CasMountLeaseRenewer"), "CAS mount renewal '{}' fenced after {} physical attempts in {} ms " - "(classification={}, confirmed_deadline_boot_ms={})", + "(classification={}, confirmed_deadline_boot_ms={}, remaining_confirmed_budget_ms={})", *context.server_root_id, context.attempts_sent, elapsedSince(context.observability_start_boot_ms, now_boot_ms), classification, - context.confirmed_deadline_boot_ms); + context.confirmed_deadline_boot_ms, + remainingConfirmedBudget(context, now_boot_ms)); } catch (...) { @@ -426,10 +430,6 @@ void deliverMountRenewObservability( } } -/// Forward declaration: defined below (same TU-unique anonymous namespace) — `allocateWriterEpoch` -/// names the current mount holder in its DecommissionRecovery live-refusal message. -String describeMountHolder(const MountLease & m); - std::optional readOwnerObject(CasOperation & op, const Layout & l, const String & server_root_id) { const auto got = op.read(l.ownerKey(server_root_id), Retry::standard()); @@ -438,17 +438,22 @@ std::optional readOwnerObject(CasOperation & op, const Layout & l, return decodeOwner(got->bytes); } -void throwIfOwnerRetired(const OwnerObject & owner, const String & srid) +void throwIfOwnerRetired(const OwnerObject & owner, const String & srid, const String & owner_key) { if (!owner.retired_at_ms) return; + if (owner.retirement_verified) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' was decommissioned (owner tombstoned at {} ms, life {}); this process cannot run " + "under the retired owner. Restart the server: a server start rejoins a finished decommission as an " + "empty member", + srid, *owner.retired_at_ms, owner.owner_life); throw Exception(ErrorCodes::CORRUPTED_DATA, - "CAS server-root '{}' was explicitly decommissioned by an operator (tombstoned at {} ms) " - "and is refusing to silently resume — if you genuinely intend to bring this server-root " - "back, manually clear the owner object's tombstone field and restart " - "(same manual-recovery pattern as an owner anchor lost over existing data)", - srid, *owner.retired_at_ms); + "CAS server-root '{}' was decommissioned (owner object '{}' retired at {} ms, owner life {}) and the " + "decommission is not finished: its retirement is not verified. Finish it with SYSTEM CAS DROP POOL MEMBER under " + "the rerun rule of the CAS docs (bringing a member back), then restart this server", + srid, owner_key, *owner.retired_at_ms, owner.owner_life); } } @@ -497,21 +502,27 @@ void deliverDeferredMountRenewObservability(uint64_t remount_attempt_no) noexcep deliverMountRenewObservability(completed, remount_attempt_no); } -bool serverRootSubtreeEmpty( +ServerRootLeftover serverRootLeftover( CasOperation & op, const Layout & l, const String & srid, const RefCatalog & catalog_observation) { const String owned_prefix = srid + "/"; for (const CatalogEntry & entry : catalog_observation.entries) if (entry.ns.string() == srid || entry.ns.string().starts_with(owned_prefix)) - return false; + return ServerRootLeftover::CatalogRows; /// Manifests and loose roots retain logical path identity. Opaque namespace stream/state debris /// alone is not evidence that this server root owns live work. if (prefixHasAnyKey(op, l.casManifestsServerPrefix(srid))) - return false; + return ServerRootLeftover::Manifests; if (prefixHasAnyKey(op, l.serverRootDataPrefix(srid))) - return false; - return true; + return ServerRootLeftover::RootsObjects; + return ServerRootLeftover::None; +} + +bool serverRootSubtreeEmpty( + CasOperation & op, const Layout & l, const String & srid, const RefCatalog & catalog_observation) +{ + return serverRootLeftover(op, l, srid, catalog_observation) == ServerRootLeftover::None; } std::optional readOwnerUuid(CasOperation & op, const Layout & l, const String & server_root_id) @@ -522,34 +533,239 @@ std::optional readOwnerUuid(CasOperation & op, const Layout & l, const return owner->server_uuid; } -void claimOwnerOrThrow( +namespace +{ +/// Every holder-originated mount body gets a random durable identity. UUIDv4 cannot be zero, but +/// keep the postcondition explicit because zero is reserved as an uninitialized in-memory value. +UInt128 newMountWriteAttemptId() +{ + const UInt128 id = UUIDHelpers::generateV4().toUnderType(); + chassert(id != UInt128{}); + return id; +} + +/// The key `IDisk::checkAccess` writes under the disk root, `clickhouse_access_check_`. +String accessProbeKey(const Layout & l, const String & srid, UInt128 server_uuid) +{ + const String hex = u128ToHex(server_uuid); + return fmt::format("{}clickhouse_access_check_{}-{}-{}-{}-{}", l.serverRootDataPrefix(srid), + hex.substr(0, 8), hex.substr(8, 4), hex.substr(12, 4), hex.substr(16, 4), hex.substr(20)); +} + +/// One `MountClaim` event per rejoin write, so the audit trail shows who took the slot and the owner. Delivery +/// failures are contained: they cannot change the rejoin decision. +void emitRejoinEvent(const CasEventSink & sink, const String & srid, std::string_view branch, std::string_view reason, + const OwnerObject & retired, UInt128 our_uuid, uint64_t new_life) noexcept +{ + try + { + if (!sink) + return; + CasEvent e; + e.type = CasEventType::MountClaim; + e.object_kind = CasEventObjectKind::None; + e.outcome = String{branch}; + e.reason = String{reason}; + e.detail = {{"server_root_id", srid}, {"branch", String{branch}}, + {"retired_uuid", u128ToHex(retired.server_uuid)}, {"retired_life", std::to_string(retired.owner_life)}, + {"server_uuid", u128ToHex(our_uuid)}, {"owner_life", std::to_string(new_life)}}; + sink(std::move(e)); + } + catch (...) // NOLINT(bugprone-empty-catch) + { + } +} + +/// The first catalog rows of the retired id's previous life, as `namespace (state)`, for the refusal text. +String leftoverCatalogRows(const RefCatalog & catalog, const String & srid) +{ + constexpr size_t max_named = 5; + const String owned_prefix = srid + "/"; + std::vector named; + size_t total = 0; + for (const CatalogEntry & entry : catalog.entries) + { + if (entry.ns.string() != srid && !entry.ns.string().starts_with(owned_prefix)) + continue; + if (++total <= max_named) + named.push_back(fmt::format("{} ({})", entry.ns.string(), nsStateToWord(entry.state))); + } + String text = fmt::format("{}", fmt::join(named, ", ")); + if (total > max_named) + text += fmt::format(" and {} more", total - max_named); + return text; +} + +/// Take over the retired `owner` as `our_uuid` at its next life. nullopt: an `If-Match` lost to another +/// write, and the caller decides again from a fresh owner read. +std::optional rejoinRetiredOwner( + CasOperation & op, const Layout & l, const String & srid, UInt128 our_uuid, const OwnerObject & owner, + const Etag & owner_etag, const ObserveRefCatalog & observe_catalog, const CasEventSink & sink) +{ + /// Checked before the retirement, so an id retired by 26.6.4 (no `epoch`, no `mount`) is never sent to a + /// rerun, which would refuse the absent counter. + const std::optional mount = op.read(l.mountKey(srid), Retry::standard()); + if (!mount || !op.head(l.epochKey(srid), Retry::standard())) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' is retired but its {} object is missing: it was retired by an older version or the " + "object was deleted by hand, so it cannot rejoin. Configure a new for this disk", + srid, mount ? "epoch" : "mount"); + /// A rerun of an unfinished decommission drains whoever owns the id when it starts. + if (!owner.retirement_verified) + throwIfOwnerRetired(owner, srid, l.ownerKey(srid)); + const MountLease slot = decodeMountLease(mount->bytes); + if (!slot.gc_fenced && !slot.isCleanRelease()) + throw Exception(ErrorCodes::ABORTED, + "CAS server-root '{}' is retired and its mount slot is held (server_uuid={}, writer_epoch={}): a " + "decommission run is still running, or its lease has not yet lapsed and been fenced by GC. Restart this " + "server later", srid, u128ToHex(slot.server_uuid), slot.writer_epoch); + + /// A late `IDisk::checkAccess` of the old life can land after the drain; nothing else of it may remain. + const String probe = accessProbeKey(l, srid, owner.server_uuid); + if (const std::optional probe_meta = op.head(probe, Retry::standard())) + /// A lost `If-Match` leaves the probe in place, and the leftover check below refuses on it. + (void)op.remove(probe, probe_meta->etag, Retry::standard()); + const RefCatalog catalog = observe_catalog(); + switch (serverRootLeftover(op, l, srid, catalog)) + { + case ServerRootLeftover::None: + break; + case ServerRootLeftover::CatalogRows: + throw Exception(ErrorCodes::ABORTED, + "CAS server-root '{}' is retired but data of its previous life remains: catalog rows of the old life are not " + "yet removed by GC: {}. Run SYSTEM CAS GC RUN '' on the GC leader (is_leader = 1 in system.cas_mounts; " + "it can take several rounds) and restart this server. A row that stays `removing` for many rounds is held " + "by GC: the leader counts it in CASGCStuckRemovals and logs a stuck-removal warning naming the reason", + srid, leftoverCatalogRows(catalog, srid)); + case ServerRootLeftover::Manifests: + throw Exception(ErrorCodes::ABORTED, + "CAS server-root '{}' is retired but data of its previous life remains: manifests under cas/manifests/{}/ are " + "not yet removed by GC. Run SYSTEM CAS GC RUN '' on the GC leader (is_leader = 1 in system.cas_mounts; " + "it can take several rounds) and restart this server", srid, srid); + case ServerRootLeftover::RootsObjects: + throw Exception(ErrorCodes::ABORTED, + "CAS server-root '{}' is retired but data of its previous life remains: an object under roots/{}/ appeared " + "after the drain. Delete it by hand and restart this server", srid, srid); + } + + /// A uuid other than the retired owner's replaces the terminal body before the owner write, so a 26.6.4 + /// starter of that owner's uuid loses its `If-Match` and has no claim recheck to fall back on. The owner's + /// own uuid writes nothing here: if its owner write then lost, its body would sit under another owner, + /// where such a starter could reclaim it. + if (our_uuid != owner.server_uuid) + { + MountLease ours = slot; + ours.server_uuid = our_uuid; + ours.hostname = getFQDNOrHostName(); + ours.pid = static_cast(::getpid()); + ours.write_attempt_id = newMountWriteAttemptId(); + ours.gc_fenced = false; + ours.min_active_build_sequence = std::numeric_limits::max(); + ours.seq += 1; + if (conflictOrThrow( + op.replace(l.mountKey(srid), encodeMountLease(ours), mount->etag, Retry::standard()), + fmt::format("CAS server-root '{}' rejoin mount write", srid))) + return std::nullopt; + emitRejoinEvent(sink, srid, "rejoin_mount_rewrite", "a rejoin under a new server uuid rewrote the retired terminal body", + owner, our_uuid, owner.owner_life + 1); + } + + const uint64_t life = owner.owner_life + 1; + if (conflictOrThrow( + op.replace( + l.ownerKey(srid), + encodeOwner(OwnerObject{.server_uuid = our_uuid, .retired_at_ms = std::nullopt, .owner_life = life}), + owner_etag, Retry::standard()), + fmt::format("CAS server-root '{}' rejoin owner write", srid))) + return std::nullopt; + emitRejoinEvent(sink, srid, "rejoin_owner", "a server start took over the retired owner at its next life", owner, our_uuid, life); + LOG_WARNING(getLogger("CasMountLease"), + "CAS server-root '{}': rejoined as an empty member under server_uuid {} at owner life {} (retired life {} of {})", + srid, u128ToHex(our_uuid), life, owner.owner_life, u128ToHex(owner.server_uuid)); + return life; +} +} + +String remountOwnerRefusal(CasOperation & op, const Layout & l, const String & srid, UInt128 our_uuid, uint64_t recorded_life) +{ + const std::optional owner = readOwnerObject(op, l, srid); + if (!owner) + return {}; + if (owner->server_uuid != our_uuid) + return fmt::format( + "CAS server-root '{}' is owned by a different server since this process started (owner server_uuid={}, " + "ours={}); this process cannot resume it", srid, u128ToHex(owner->server_uuid), u128ToHex(our_uuid)); + if (owner->retired_at_ms) + return fmt::format( + "CAS server-root '{}' was decommissioned while this process was fenced (owner retired at {} ms, owner " + "life {}); this process stops for good and a restart refuses this server_root_id, see the CAS " + "troubleshooting docs: a decommissioned member refuses to start", + srid, *owner->retired_at_ms, owner->owner_life); + if (owner->owner_life != recorded_life) + return fmt::format( + "CAS server-root '{}': the owner moved from life {} to life {} since this process started (a rejoin or " + "a decommission rollback); restart the server to continue under the current life", + srid, recorded_life, owner->owner_life); + return {}; +} + +uint64_t claimOwnerOrThrow( CasOperation & op, const Layout & l, const String & srid, UInt128 our_uuid, - const ObserveRefCatalog & observe_catalog) + const ObserveRefCatalog & observe_catalog, RetiredOwnerPolicy retired_policy, const CasEventSink & sink) { if (!observe_catalog) throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS server-root '{}': catalog observer is required", srid); const String key = l.ownerKey(srid); - /// Owner present → it is identity: equal UUID is ok, a different UUID fails closed regardless - /// of any lease/clock state. - if (const std::optional owner = readOwnerObject(op, l, srid)) + /// A lost rejoin write decides again from a fresh read; an owner or slot that changes on every decision fails + /// the start instead of spinning. + constexpr size_t max_owner_decisions = 4; + const bool may_rejoin = retired_policy == RetiredOwnerPolicy::RejoinIfVerified; + for (size_t decision = 1;; ++decision) { - if (owner->server_uuid == our_uuid) + const std::optional current = op.read(key, Retry::standard()); + if (!current) + break; + const OwnerObject owner = decodeOwner(current->bytes); + if (owner.retired_at_ms && may_rejoin) { - throwIfOwnerRetired(*owner, srid); - return; + if (our_uuid == UInt128{}) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS server-root '{}' is retired (owner tombstoned at {} ms, life {}) and this process has no " + "server uuid, so it does not rejoin it. Only a server start with a server uuid rejoins a retired " + "id: start the server that should own it, or configure a new for this disk", + srid, *owner.retired_at_ms, owner.owner_life); + if (const std::optional life = rejoinRetiredOwner(op, l, srid, our_uuid, owner, current->etag, observe_catalog, sink)) + return *life; + if (decision == max_owner_decisions) + throw Exception(ErrorCodes::ABORTED, + "CAS server-root '{}': the owner or the mount slot kept changing during {} rejoin attempts; " + "restart to try again", srid, max_owner_decisions); + continue; + } + /// Owner present → it is identity: equal UUID is ok, a different UUID fails closed regardless + /// of any lease/clock state. + if (owner.server_uuid == our_uuid) + { + if (retired_policy != RetiredOwnerPolicy::AcceptForDecommission) + throwIfOwnerRetired(owner, srid, key); + return owner.owner_life; } - /// Mirror mountDoubleStartMessage's operator guidance: the by-far most common cause is a - /// REGENERATED local ClickHouse uuid file (wiped /var/lib/clickhouse, a pod rescheduled - /// without a persistent volume) while the pool kept the old identity — name it and the - /// recovery options instead of a bare refusal. + /// The common cause is a regenerated local uuid file (wiped /var/lib/clickhouse, a pod recreated + /// without a persistent volume) while the pool kept the old identity; name it and the supported + /// ways out, with the hand delete last: it is refused anyway while the subtree holds data. throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS server-root '{}' is owned by a different server (owner server_uuid={}, ours={}) — refusing to claim. " - "This usually means THIS server's local uuid file was regenerated (e.g. /var/lib/clickhouse was wiped, " - "or the container/pod was recreated without a persistent volume) while the pool kept the old identity. " - "Recover by restoring the old local uuid file; or configure a fresh for this disk; " - "or — only after verifying that NO server uses this root — manually delete the owner object '{}' and restart.", - srid, u128ToHex(owner->server_uuid), u128ToHex(our_uuid), key); + "Either this server's local uuid file was regenerated (/var/lib/clickhouse wiped, or a pod recreated " + "without a persistent volume) while the pool kept the old identity, or two servers are configured with " + "the same . Recover by restoring the old uuid file, or by starting this " + "server under a new . To reclaim the old id's data once its owning server is " + "permanently gone, decommission it — SYSTEM CAS DROP POOL MEMBER '{}' FROM DISK '' on a " + "surviving member (offline: clickhouse-disks cas-drop-member) — once its slot in system.cas_mounts is " + "'terminated' or 'fenced'. " + "Deleting the owner object '{}' by hand is a last resort, only after verifying that NO server uses this " + "root; it is refused while the subtree still holds data.", + srid, u128ToHex(owner.server_uuid), u128ToHex(our_uuid), srid, key); } /// Owner absent. Claiming is allowed ONLY over a provably-empty subtree; an absent owner over @@ -557,15 +773,16 @@ void claimOwnerOrThrow( if (!serverRootSubtreeEmpty(op, l, srid, observe_catalog())) throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS server-root '{}' has no owner anchor but its data subtree is non-empty " - "(identity lost over existing data) — refusing to re-claim", + "(identity lost over existing data) — refusing to re-claim. Restoring the local uuid file does not help, " + "since no owner records a uuid to match. Configure a new for this disk", srid); const std::optional occupant = conflictOrThrow( - op.create(key, encodeOwner(OwnerObject{.server_uuid = our_uuid, .retired_at_ms = std::nullopt}), + op.create(key, encodeOwner(OwnerObject{.server_uuid = our_uuid, .retired_at_ms = std::nullopt, .owner_life = 1}), Retry::standard()), fmt::format("CAS server-root '{}' owner claim", srid)); if (!occupant) - return; + return 1; /// The conditional create conflicted. Recompute the whole catalog + manifest + roots bundle; /// no stale emptiness result is carried across the conflict. @@ -586,8 +803,9 @@ void claimOwnerOrThrow( "CAS server-root '{}' owner anchor vanished during claim", srid); if (reread->server_uuid == our_uuid) { - throwIfOwnerRetired(*reread, srid); - return; + if (retired_policy != RetiredOwnerPolicy::AcceptForDecommission) + throwIfOwnerRetired(*reread, srid, key); + return reread->owner_life; } throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS server-root '{}' was claimed by a different server during our claim (foreign owner) " @@ -596,7 +814,7 @@ void claimOwnerOrThrow( } uint64_t allocateWriterEpoch( - CasOperation & op, const Layout & l, const String & srid, EpochMintPolicy policy, uint64_t now_ms, + CasOperation & op, const Layout & l, const String & srid, const ObserveRefCatalog & observe_catalog) { if (!observe_catalog) @@ -646,40 +864,12 @@ uint64_t allocateWriterEpoch( break; /// authoritative absence — fresh-root bootstrap proceeds below case ProbeOutcome::Present: { - if (policy == EpochMintPolicy::DecommissionRecovery) - { - chassert(now_ms != 0); /// the decommission caller must pass its clock - const MountLease surviving = decodeMountLease(*mount_probe.body); - /// Deliberately weaker than claimMount's reclaim gate, which never trusts a - /// bare wall-clock comparison alone (only gc_fenced / the clean-farewell - /// min_active_build_sequence==UINT64_MAX marker / a caller-proven-dead - /// incarnation justify a reclaim there, because clock skew can misjudge - /// liveness). This is still safe: (a) the mint below is DISTINCT from the - /// survivor's epoch by construction, so no same-(uuid, epoch) pair is ever - /// representable even if this liveness read is wrong; (b) claimMount right - /// after this still applies its own STRONG liveness gate and refuses a - /// genuinely live member regardless of what happens here. So a clock-skewed - /// "terminal" misread can only burn one epoch number on a doomed - /// decommission attempt that aborts at claimMount — it can never admit a - /// claim over a live member. - const bool live = !surviving.gc_fenced && surviving.expires_at_ms > now_ms; - if (live) - throw Exception(ErrorCodes::ABORTED, - "CAS decommission '{}': epoch object missing but a LIVE mount lease " - "exists ({}) — refusing to re-mint an epoch under a live member " - "(stop the server or wait for its lease to lapse)", - srid, describeMountHolder(surviving)); - /// Terminal mount: proceed, but mint an epoch DISTINCT from the survivor's - /// by construction — the same-pair state is unrepresentable on this path. - current.next_writer_epoch = std::max(1, surviving.writer_epoch + 1); - break; - } throw Exception(ErrorCodes::CORRUPTED_DATA, "CAS server-root '{}' has no durable epoch object but a mount lease exists — " "durable epoch state was lost while a mount is live or recently live; " - "refusing to re-mint epoch 1. If no server is live on this root, " - "decommission it or manually remove the stale mount object '{}'.", - srid, l.mountKey(srid)); + "refusing to re-mint epoch 1. If no server uses this root, start with a new " + "server_root_id; a decommission never recreates a counter", + srid); } case ProbeOutcome::ContainerAbsent: case ProbeOutcome::AccessDenied: @@ -695,7 +885,8 @@ uint64_t allocateWriterEpoch( } allocated = current.next_writer_epoch; - return encodeServerEpoch(ServerEpoch{.next_writer_epoch = allocated + 1}); + return encodeServerEpoch( + ServerEpoch{.next_writer_epoch = allocated + 1, .allocation_nonce = UUIDHelpers::generateV4().toUnderType()}); }, Retry::standard()); @@ -709,15 +900,6 @@ uint64_t allocateWriterEpoch( namespace { -/// Every holder-originated mount body gets a random durable identity. UUIDv4 cannot be zero, but -/// keep the postcondition explicit because zero is reserved as an uninitialized in-memory value. -UInt128 newMountWriteAttemptId() -{ - const UInt128 id = UUIDHelpers::generateV4().toUnderType(); - chassert(id != UInt128{}); - return id; -} - /// Build a fresh mount-lease body for (uuid, epoch) with the given seq, stamped from `now_ms`. MountLease makeMountBody(UInt128 uuid, uint64_t epoch, uint64_t seq, uint64_t now_ms, uint64_t ttl_ms) { @@ -782,12 +964,35 @@ void emitMountEvent(const CasEventSink & sink, CasEventType type, const String & /// allocate or recurse through the same diagnostic machinery at a protocol-critical site. } } + +/// Why `owner` is not the one `expected` names; nullopt when it is. +std::optional ownerRefusal(const std::optional & owner, const String & srid, const OwnerExpectation & expected) +{ + if (!owner) + return fmt::format("CAS server-root '{}': the owner anchor vanished after the mount claim", srid); + if (owner->server_uuid != expected.server_uuid) + return fmt::format( + "CAS server-root '{}' is owned by a different server since this claim began (owner server_uuid={}, " + "ours={}); the mount slot was handed to another server and this claim is withdrawn", + srid, u128ToHex(owner->server_uuid), u128ToHex(expected.server_uuid)); + if (owner->retired_at_ms && expected.retired != RetiredOwnerPolicy::AcceptForDecommission) + return fmt::format( + "CAS server-root '{}' was decommissioned (tombstoned at {} ms) while this claim was in progress; " + "refusing to run under a retired owner", srid, *owner->retired_at_ms); + if (owner->owner_life != expected.owner_life) + return fmt::format( + "CAS server-root '{}': the owner moved from life {} to life {} since this process started (a " + "retirement, a rollback or a rejoin happened in between); restart to continue under the current life", + srid, expected.owner_life, owner->owner_life); + return std::nullopt; +} } MountClaimResult claimMount( CasOperation & op, const Layout & l, const String & srid, UInt128 our_uuid, uint64_t our_epoch, uint64_t now_ms, uint64_t ttl_ms, const std::optional & proven_dead_incarnation, - const CasEventSink & sink, const std::optional & unsafe_reclaim_authorization) + const CasEventSink & sink, const std::optional & unsafe_reclaim_authorization, + const std::optional & foreign_reclaim_owner) { const String key = l.mountKey(srid); const auto got = op.read(key, Retry::standard()); @@ -808,19 +1013,40 @@ MountClaimResult claimMount( const MountLease existing = decodeMountLease(got->bytes); - /// Foreign owner → fail closed regardless of expiry. (This runs after the owner gate, so a foreign - /// mount should not normally exist, but the lease must never be taken across UUIDs.) + const bool clean_marker = existing.isCleanRelease(); + + /// A live body of another uuid is never taken. A terminal one below our epoch is what a rejoin under a new + /// uuid finds; it is reclaimed only once the owner reads as this claimer's own life. The owner is read before + /// the write, not only rechecked after it, so a reclaim never lands under somebody else's owner. if (existing.server_uuid != our_uuid) { - emitMountEvent(sink, CasEventType::MountConflict, srid, "foreign_owner", &existing, - "mount slot is held by a foreign server_uuid — refusing to take over across identities"); - return {.kind = MountClaimResult::ForeignOwner, .body = existing, .etag = std::nullopt}; + const bool reclaimable = (existing.gc_fenced || clean_marker) && existing.writer_epoch < our_epoch + && foreign_reclaim_owner && !ownerRefusal(readOwnerObject(op, l, srid), srid, *foreign_reclaim_owner); + if (!reclaimable) + { + emitMountEvent(sink, CasEventType::MountConflict, srid, "foreign_owner", &existing, + "mount slot is held by a foreign server_uuid — refusing to take over across identities"); + return {.kind = MountClaimResult::ForeignOwner, .body = existing, .etag = std::nullopt}; + } + } + + /// An older allocation that reached the slot late must not replace a newer incarnation of this + /// server: recovery seals only epochs below the live one, so a late write of the newer epoch would + /// never be sealed out. + if (existing.writer_epoch > our_epoch) + { + emitMountEvent(sink, CasEventType::MountConflict, srid, "newer_epoch", &existing, + "same server_uuid, newer writer_epoch in the mount slot — refusing to replace it"); + throw Exception(ErrorCodes::ABORTED, + "CAS mount slot '{}' holds writer_epoch {} of this server_uuid, newer than the writer_epoch {} " + "this claim allocated: another start or a decommission of server_root_id '{}' allocated after " + "this one. Refusing to replace the newer incarnation ({})", + key, existing.writer_epoch, our_epoch, srid, describeMountHolder(existing)); } - /// Same uuid + same epoch: it is OUR OWN claim — but a FENCED body is terminal for this - /// (uuid, epoch): the GC dropped its ack from the floor when it fenced. Refreshing it in place - /// would reactivate a fenced incarnation — a fence permanently consumes this `(server_uuid, - /// writer_epoch)` pair, so the caller must re-open with a fresh `writer_epoch`. + /// Our own `(uuid, epoch)` is in the slot, but this call has written nothing: the engine settles a + /// sent attempt inside the write itself. Another process was allocated the same epoch, and its body + /// is never replaced. A fenced one is terminal for the pair; the caller takes a fresh epoch. if (existing.writer_epoch == our_epoch) { if (existing.gc_fenced) @@ -830,20 +1056,12 @@ MountClaimResult claimMount( "recover with a fresh writer_epoch"); return {.kind = MountClaimResult::FencedSelf, .body = existing, .etag = std::nullopt}; } - const MountLease body = makeMountBody(our_uuid, our_epoch, existing.seq + 1, now_ms, ttl_ms); - if (const std::optional raced - = conflictOrThrow(op.replace(key, encodeMountLease(body), got->etag, Retry::standard()), - fmt::format("CAS mount slot refresh of '{}'", key))) - /// The mount changed under us between the read and the write, so `got->etag` is KNOWN - /// STALE -- that mismatch is exactly why the write was refused. What the write's resolve - /// read observed is current, and it is that pair that is reported. - return racedDoubleStart(*raced); - emitMountEvent(sink, CasEventType::MountClaim, srid, "refresh", &existing, - "own claim replayed — refreshed seq + expiry"); - return {.kind = MountClaimResult::Claimed, .body = body, .etag = std::nullopt}; + emitMountEvent(sink, CasEventType::MountConflict, srid, "same_epoch_occupied", &existing, + "own (uuid, epoch) is already in the mount slot, written by another process — refusing to write"); + return {.kind = MountClaimResult::LiveDoubleStart, .body = existing, .etag = got->etag}; } - /// Same uuid, DIFFERENT epoch: reclaim ONLY on a certificate of death that needs no fresh + /// Same uuid, OLDER epoch: reclaim ONLY on a certificate of death that needs no fresh /// wall-clock trust — never by comparing `expires_at_ms` against `now_ms`: /// - `gc_fenced` → the fence-out is terminal for that incarnation by construction (its renewer's /// every renewal fails the token guard forever, so it can never write again) — there is no @@ -863,7 +1081,6 @@ MountClaimResult claimMount( /// clean-marked, not (yet) proven-dead, not unsafe-authorized lease may simply be a live twin, and /// `expires_at_ms` alone can never distinguish that from a dead predecessor across two different /// clocks. - const bool clean_marker = existing.min_active_build_sequence == std::numeric_limits::max(); const bool proven_dead = proven_dead_incarnation && *proven_dead_incarnation == got->etag; const bool unsafe_authorized = unsafe_reclaim_authorization && *unsafe_reclaim_authorization == got->etag; if (existing.gc_fenced || clean_marker || proven_dead || unsafe_authorized) @@ -881,7 +1098,8 @@ MountClaimResult claimMount( : proven_dead ? MountPriorState::UncleanObserved : MountPriorState::UncleanUnsafe; emitMountEvent(sink, CasEventType::MountClaim, srid, "reclaim", &existing, - existing.gc_fenced ? "same server_uuid, different writer_epoch, GC-fenced — reclaimed" + existing.server_uuid != our_uuid ? "terminal body of another server_uuid, the owner is ours — reclaimed" + : existing.gc_fenced ? "same server_uuid, different writer_epoch, GC-fenced — reclaimed" : clean_marker ? "same server_uuid, different writer_epoch, clean farewell — reclaimed" : proven_dead ? "same server_uuid, different writer_epoch, observed dead by " "token-stability observation — reclaimed" @@ -900,6 +1118,55 @@ MountClaimResult claimMount( return {.kind = MountClaimResult::LiveDoubleStart, .body = existing, .etag = got->etag}; } +namespace +{ +/// Rewrite the claimer's own live body `gc_fenced`, preserving everything else as the GC fence does. Only +/// `(our_uuid, our_epoch)`, not yet fenced or farewelled, is touched; anything else in the slot is left alone. +void fenceOwnClaim(CasOperation & op, const Layout & l, const String & srid, UInt128 our_uuid, uint64_t our_epoch, + const CasEventSink & sink) +{ + const String key = l.mountKey(srid); + WriteResult fenced = op.readModifyWrite(key, + [&](const std::optional & observed) -> std::optional + { + if (!observed) + return std::nullopt; + const MountLease body = decodeMountLease(observed->bytes); + if (!body.isHeldBy(our_uuid, our_epoch) || body.gc_fenced || body.isCleanRelease()) + return std::nullopt; + return encodeMountLease(body.fencedCopy()); + }, + Retry::standard()); + if (std::holds_alternative(fenced)) + emitMountEvent(sink, CasEventType::MountRelease, srid, "self_fenced", nullptr, + "the owner recheck refused this claim; the slot is fenced so a successor need not wait"); + else if (!std::holds_alternative(fenced)) + LOG_WARNING(getLogger("CasMountLease"), + "CAS server-root '{}': the refused claim's mount body could not be fenced; its lease expires on its own", srid); +} +} + +void recheckOwnerAfterClaim( + CasOperation & op, const Layout & l, const String & srid, const OwnerExpectation & expected, + uint64_t our_epoch, const CasEventSink & sink) +{ + const std::optional refusal = ownerRefusal(readOwnerObject(op, l, srid), srid, expected); + if (!refusal) + return; + + /// The fence only spares a successor the lease wait; its failure must not replace the owner's reason. + try + { + fenceOwnClaim(op, l, srid, expected.server_uuid, our_epoch, sink); + } + catch (...) + { + tryLogCurrentException(getLogger("CasMountLease"), + fmt::format("CAS server-root '{}': the refused claim's mount body could not be fenced", srid)); + } + throw Exception(ErrorCodes::CORRUPTED_DATA, "{}", *refusal); +} + String mountDoubleStartMessage(const String & srid, const std::optional & existing) { const String identity = existing @@ -918,9 +1185,8 @@ String mountDoubleStartMessage(const String & srid, const std::optional::max(); + return threshold_ms; } MountClaimResult claimMountAwaitingExpiry( @@ -944,18 +1223,15 @@ MountClaimResult claimMountAwaitingExpiry( uint64_t ttl_ms, uint64_t poll_interval_ms, const std::function & sleep_ms_fn, const std::function & on_wait_start, - const CasEventSink & sink) + const CasEventSink & sink, + const std::optional & foreign_reclaim_owner) { /// A zero poll interval would spin; a single-ms floor keeps the loop a real (bounded) wait. const uint64_t poll = poll_interval_ms == 0 ? 1 : poll_interval_ms; - /// Rate-bound observation threshold: the full lease TTL, plus a 5% allowance for clock-rate - /// mismatch between the holder's and our own local clock, plus one poll interval for observation - /// discreteness. It is measured only with OUR OWN clock (`mono_ms_fn`); no cross-node wall-clock - /// comparison participates in this loop. `poll` here is half the renewal period (the caller's own - /// poll cadence), so this threshold is close to, but not identical to, GC's heartbeat fence-out - /// threshold, which passes the full renewal period into the same shared helper. - const uint64_t threshold_ms = mountObservationThresholdMs(ttl_ms, poll); + /// Set when an observation starts, from the observed body; one token is one body, so the threshold + /// of an observation never changes. + uint64_t threshold_ms = 0; std::optional observed; uint64_t observed_since = 0; @@ -965,9 +1241,16 @@ MountClaimResult claimMountAwaitingExpiry( { const bool threshold_met = observed && mono_ms_fn() - observed_since >= threshold_ms; MountClaimResult r = claimMount(op, l, srid, our_uuid, our_epoch, now_ms_fn(), ttl_ms, - threshold_met ? observed : std::nullopt, sink, /*unsafe_reclaim_authorization=*/{}); + threshold_met ? observed : std::nullopt, sink, /*unsafe_reclaim_authorization=*/{}, foreign_reclaim_owner); if (r.kind != MountClaimResult::LiveDoubleStart) return r; + /// Observation certifies only an older epoch dead; a body of our own pair is never replaced, + /// so waiting on it could not end. + if (r.body && r.body->isHeldBy(our_uuid, our_epoch)) + return r; + /// A raced write observed a newer epoch of ours; the re-read refuses it with the newer-epoch error. + if (r.body && r.body->server_uuid == our_uuid && r.body->writer_epoch > our_epoch) + return claimMount(op, l, srid, our_uuid, our_epoch, now_ms_fn(), ttl_ms, std::nullopt, sink, {}, foreign_reclaim_owner); /// `claimMount` already read the current body, and a raced write reports whatever its own /// resolve read observed. Reuse `r.etag` whenever it is set instead of re-reading the SAME key @@ -980,14 +1263,9 @@ MountClaimResult claimMountAwaitingExpiry( const auto got = op.read(l.mountKey(srid), Retry::standard()); if (!got) { - /// The slot vanished between claimMount's own GET and ours — normally self-resolving - /// within one more `claimMount` call (which re-mints fresh on an absent slot), but under - /// slot churn (something else concurrently removing/re-minting it) that resolution could - /// keep losing the same race. Pace this like every other iteration and - /// count it toward the SAME bounded restart budget the incarnation-churn case below - /// uses, instead of spinning read/claim/write at backend RTT with no sleep and no bound - /// — a persistently vanishing slot is exactly as "alive and contended" as a persistently - /// renewing holder. + /// The slot vanished between `claimMount`'s read and ours (a 26.6.4 decommission or a hand delete): + /// the next `claimMount` re-mints it. Paced and counted like a renewing holder, so a slot that + /// keeps vanishing cannot spin read, claim and write at backend RTT. if (++restarts > kMaxObservationRestarts) return r; sleep_ms_fn(poll); @@ -1005,7 +1283,10 @@ MountClaimResult claimMountAwaitingExpiry( return r; observed = *current_etag; observed_since = mono_ms_fn(); - if (on_wait_start && r.body) + if (!r.body) + throw Exception(ErrorCodes::LOGICAL_ERROR, "CAS mount '{}': observed a mount token without its body", srid); + threshold_ms = mountObservationThresholdMs(ttl_ms, declaredMountLeaseMs(*r.body), poll); + if (on_wait_start) on_wait_start(*r.body, threshold_ms); LOG_INFO(getLogger("CasMountLease"), "Attempting to mount content-addressed server root {} after node change or hard " @@ -1022,23 +1303,34 @@ bool MountIncarnationObservation::heldFor(uint64_t threshold_ms, uint64_t sample return sample_ms >= first_seen_mono_ms && sample_ms - first_seen_mono_ms >= threshold_ms; } +namespace +{ +/// The setting is local, so different values are legal; a longer one slows recovery after its holder's +/// crash, which the operator should know about. +void warnOnceIfHolderLeaseDiffers( + const String & srid, const MountLease & holder, uint64_t observer_lease_ms, std::set & warned) +{ + const uint64_t declared_ms = declaredMountLeaseMs(holder); + if (declared_ms == 0 || declared_ms == observer_lease_ms || !warned.insert(srid).second) + return; + LOG_WARNING(getLogger("CasHeartbeatFloor"), + "This server is configured with cas_mount_lease_ttl_ms={} but the holder of slot {} (hostname={}, pid={}) " + "declares a lease of {} ms. Observers wait for the longer of the two; a longer lease on one server " + "delays recovery after that server's crash.", + observer_lease_ms, srid, holder.hostname, holder.pid, declared_ms); +} +} + HeartbeatFloor computeHeartbeatFloor(CasOperation & op, const Layout & l, uint64_t now_ms, const std::function & mono_ms_fn, - uint64_t stable_threshold_ms, MountObservationMap & obs) + uint64_t observer_lease_ms, uint64_t cadence_ms, MountObservationMap & obs, + std::set & lease_mismatch_warned) { HeartbeatFloor floor; /// Taken before any read: it can confirm that a token held, never date a sighting. const uint64_t round_start_ms = mono_ms_fn(); - /// `obs` is keyed by every srid this leader has EVER observed, but a - /// srid removed from the LIST entirely (its `/mount` key gone -- e.g. `SYSTEM CAS - /// DROP POOL MEMBER`) is never visited by the walk below again, so its entry would otherwise linger - /// forever (~150-250 B/srid, worse on a long-lived leader across many decommissions). Track every - /// srid actually seen THIS pass and prune anything else out of `obs` at the end -- disjoint from the - /// mid-walk `obs.erase(srid)` calls below (those fire for a srid seen but now terminal/fenced/gone - /// this pass; this is for a srid not seen AT ALL). std::set seen_srids; - const String prefix = l.serverRootsPrefix(); op.forEachListedKey(prefix, [&](const ListedKey & listed) { @@ -1059,6 +1351,7 @@ HeartbeatFloor computeHeartbeatFloor(CasOperation & op, const Layout & l, uint64 /// holder's own renewal installed, and the observation check below then sees the new /// incarnation and restarts the window -- which counts the slot `live` and declines the write. /// That is why no arm that counts a slot ever also asks for a fence-out body. + uint64_t threshold_ms = 0; WriteResult fenced_out = op.readModifyWrite(key, [&](const std::optional & observed) -> std::optional { @@ -1076,13 +1369,16 @@ HeartbeatFloor computeHeartbeatFloor(CasOperation & op, const Layout & l, uint64 obs.erase(srid); /// terminal — no further observation needed return std::nullopt; } - if (m.min_active_build_sequence == std::numeric_limits::max()) + if (m.isCleanRelease()) { ++floor.terminated; obs.erase(srid); /// terminal — no further observation needed return std::nullopt; } + warnOnceIfHolderLeaseDiffers(srid, m, observer_lease_ms, lease_mismatch_warned); + threshold_ms = mountObservationThresholdMs(observer_lease_ms, declaredMountLeaseMs(m), cadence_ms); + /// Observation-based liveness: stable ONLY if the SAME incarnation was already being /// watched and has now held for the full threshold on our OWN monotonic clock. Anything /// else — no prior observation, or a changed incarnation (a live renewal, including one @@ -1090,7 +1386,7 @@ HeartbeatFloor computeHeartbeatFloor(CasOperation & op, const Layout & l, uint64 /// counts as `live` this call. const auto it = obs.find(srid); const bool stable = it != obs.end() && it->second.etag == observed->etag - && it->second.heldFor(stable_threshold_ms, round_start_ms); + && it->second.heldFor(threshold_ms, round_start_ms); if (!stable) { @@ -1103,10 +1399,7 @@ HeartbeatFloor computeHeartbeatFloor(CasOperation & op, const Layout & l, uint64 /// Stable past the threshold, not yet fenced → fence-out preserving the whole body /// (gc_fenced = true, seq + 1) against the incarnation this decision observed. - MountLease fenced = m; - fenced.gc_fenced = true; - fenced.seq = m.seq + 1; - return encodeMountLease(fenced); + return encodeMountLease(m.fencedCopy()); }, Retry::standard()); @@ -1119,7 +1412,7 @@ HeartbeatFloor computeHeartbeatFloor(CasOperation & op, const Layout & l, uint64 "CAS GC fenced out mount lease for content-addressed server root {} at " "wall-clock ms {}: its write incarnation held unchanged for >= {} ms on the GC " "leader's own monotonic clock (token-stability observation)", - srid, now_ms, stable_threshold_ms); + srid, now_ms, threshold_ms); return true; } /// Declined: the decision above already classified and counted this slot, and asked for no @@ -1130,9 +1423,10 @@ HeartbeatFloor computeHeartbeatFloor(CasOperation & op, const Layout & l, uint64 return true; }, Retry::standard()); - /// Prune every `obs` entry for a srid this pass's walk never saw at all. - for (auto it = obs.begin(); it != obs.end(); ) - it = seen_srids.contains(it->first) ? std::next(it) : obs.erase(it); + /// A slot deleted by a 26.6.4 decommission or by hand is never listed again; without this its srid would + /// stay in both maps forever on a long-lived leader. + std::erase_if(obs, [&](const auto & entry) { return !seen_srids.contains(entry.first); }); + std::erase_if(lease_mismatch_warned, [&](const String & s) { return !seen_srids.contains(s); }); return floor; } @@ -1174,7 +1468,7 @@ std::vector probeNonTerminalMountSlots(CasOperation & op, return true; } - if (m.gc_fenced || m.min_active_build_sequence == std::numeric_limits::max()) + if (m.gc_fenced || m.isCleanRelease()) return true; /// terminal: fenced out by GC, or the holder's own graceful farewell. slots.push_back(NonTerminalMountSlot{srid, fmt::format( @@ -1216,12 +1510,33 @@ std::vector listMounts(CasOperation & op, const Layout & layout, uint } if (info.lease.gc_fenced) info.state = "fenced"; - else if (info.lease.min_active_build_sequence == std::numeric_limits::max()) + else if (info.lease.isCleanRelease()) info.state = "terminated"; else if (now_ms <= info.lease.expires_at_ms + skew_margin_ms) info.state = "live"; else info.state = "expired"; + /// A decommission keeps the slot; the owner says whether the member behind a terminal row is retired, and + /// whether that retirement is verified: a rerun is for an unverified one only. + if (info.state == "terminated" || info.state == "fenced") + { + if (const auto owner_got = op.read(layout.ownerKey(info.srid), Retry::standard())) + { + try + { + const OwnerObject owner = decodeOwner(owner_got->bytes); + if (owner.retired_at_ms) + { + info.state = "retired"; + info.retirement_verified = owner.retirement_verified; + } + } + catch (...) + { + info.state = "corrupt"; + } + } + } out.push_back(std::move(info)); return true; }, Retry::standard()); @@ -1246,7 +1561,7 @@ FenceCertificate classifyFenceCertificate(const MountLease & lease, uint64_t fen { if (lease.gc_fenced) return FenceCertificate::GcFenced; - if (lease.min_active_build_sequence == std::numeric_limits::max()) + if (lease.isCleanRelease()) return FenceCertificate::CleanFarewell; if (lease.writer_epoch != fence_writer_epoch) return FenceCertificate::SupersededEpoch; @@ -1462,6 +1777,7 @@ uint64_t MountLeaseRenewer::start(Liveness liveness) seq = 1; last_etag = etag; + last_body = body; last_committed_attempt_start_boot_ms = attempt_start_boot_ms; const uint64_t ttl_ms = static_cast(ttl.count()); confirmed_deadline_boot_ms = attempt_start_boot_ms > std::numeric_limits::max() - ttl_ms @@ -1568,16 +1884,49 @@ MountRenewResult MountLeaseRenewer::terminalResult(MountRenewResult result) MountRenewResult MountLeaseRenewer::renew(const MountRenewOperationEnvironment & environment) { - return renewOn(mount_requests, environment); + return renewOn(mount_requests, environment, /*until_definitive=*/false); } MountRenewResult MountLeaseRenewer::renewForRemount(const MountRenewOperationEnvironment & environment) { - return renewOn(open_requests, environment); + return renewOn(open_requests, environment, /*until_definitive=*/false); +} + +MountRenewResult MountLeaseRenewer::renewUntilDefinitive(const MountRenewOperationEnvironment & environment) +{ + return renewOn(open_requests, environment, /*until_definitive=*/true); +} + +Observation MountLeaseRenewer::readUntilObserved( + CasOperation & op, const Retry & policy, const MountRenewOperationEnvironment & environment, + uint64_t write_started_boot_ms) const +{ + const auto boot_clock = environment.boot_ms ? environment.boot_ms : boot_ms_fn; + uint64_t pause_ms = Retry::pauseAfterAttempt(*policy.attempt_spacing_ms, write_started_boot_ms, boot_clock()); + while (op.admitted()) + { + op.pause(pause_ms); + try + { + std::optional got = op.read(key, policy); + if (!got) + return ProvenAbsent{}; + return std::move(*got); + } + catch (const Exception & e) + { + if (isDeterministicLocalFailure(e.code())) + throw; + /// Without a deadline the read repeats a transport fault itself, so what reaches here is + /// the store's refusal or a stop, which `op.admitted` tells apart. + pause_ms = environment.refusal_pause_ms; + } + } + return NotObserved{}; } MountRenewResult MountLeaseRenewer::renewOn( - CasRequests & plane, const MountRenewOperationEnvironment & environment) + CasRequests & plane, const MountRenewOperationEnvironment & environment, bool until_definitive) { const MountRenewObservabilityRegistration observability_registration = beginMountRenewObservabilityCall(); const MountRenewObservabilityCallGuard observability_guard(observability_registration); @@ -1590,11 +1939,6 @@ MountRenewResult MountLeaseRenewer::renewOn( static_cast(renewer_state)); const auto boot_clock = environment.boot_ms ? environment.boot_ms : boot_ms_fn; - /// Sampled BEFORE the write. A refused admission is reported as "never attempted" only when this - /// node had already been asked to stop, and reading the flag afterwards could not tell that apart - /// from a flag the refusal itself set. - const bool cancelled = environment.cancelled && environment.cancelled(); - const uint64_t wall_ms = now_ms_fn(); const uint64_t attempt_start_boot_ms = boot_clock(); const uint64_t next_seq = seq + 1; @@ -1617,17 +1961,69 @@ MountRenewResult MountLeaseRenewer::renewOn( result.attempt_start_boot_ms = attempt_start_boot_ms; CasOperation op = plane.admit(environment.live); + /// Counted here rather than taken from one engine call: the background renewal may span several. + uint32_t attempts_sent = 0; + uint64_t last_attempt_start_boot_ms = attempt_start_boot_ms; + op.observeRequests(RequestObserver{ + .on_write_attempt = [&](uint32_t) + { + ++attempts_sent; + last_attempt_start_boot_ms = boot_clock(); + if (environment.requests.on_write_attempt) + environment.requests.on_write_attempt(attempts_sent); + }, + .on_failure = environment.requests.on_failure, + }); + const Etag expected = precondition(); + const Retry policy = until_definitive + ? Retry::untilDefinitive() + : Retry::untilLeaseSafe(confirmed_deadline_boot_ms, static_cast(lease_safety_margin.count())); std::optional written; try { - written = op.replace(key, body, precondition(), - Retry::untilLeaseSafe(confirmed_deadline_boot_ms, static_cast(lease_safety_margin.count()))); + written = op.replace(key, body, expected, policy); + while (until_definitive) + { + uint64_t pause_ms = 0; + if (std::holds_alternative(*written)) + { + /// `writeLoop` refuses only when no attempt of the call is unclear, so no copy of ours + /// can land behind a new call with the same tuple. + pause_ms = environment.refusal_pause_ms; + } + else if (Conflict * conflict = std::get_if(&*written)) + { + if (std::holds_alternative(conflict->seen)) + conflict->seen = readUntilObserved(op, policy, environment, last_attempt_start_boot_ms); + if (std::holds_alternative(conflict->seen)) + { + /// Counted as sent: the refused precondition says the slot may have moved, so a farewell + /// naming our last token could meet a holder entitled to it. + written = GaveUp{GaveUp::Why::FenceLost, GaveUp::Source::Policy, /*sent_any=*/true, NotObserved{}, attempts_sent}; + break; + } + /// Our last confirmed body under the token we named: the slot did not move. A new call is + /// safe because a call with an unclear attempt returns `Conflict` only once the token + /// moved, so this call left no copy of ours that could land. + const Object * seen = std::get_if(&conflict->seen); + if (!seen || seen->etag != expected || seen->bytes != last_body) + break; + pause_ms = Retry::pauseAfterAttempt(*policy.attempt_spacing_ms, last_attempt_start_boot_ms, boot_clock()); + } + else + { + break; + } + op.pause(pause_ms); + written = op.replace(key, body, expected, policy); + } } catch (...) { /// The engine surfaces a deterministic local failure unchanged rather than reissuing it. markMountRenewTermination(MountRenewTerminalClassification::DeterministicFailure); result.failure = std::current_exception(); + result.attempts_sent = attempts_sent; return terminalResult(std::move(result)); } @@ -1635,22 +2031,23 @@ MountRenewResult MountLeaseRenewer::renewOn( { seq = next_seq; last_etag = std::move(committed->etag); + last_body = body; last_committed_attempt_start_boot_ms = attempt_start_boot_ms; const uint64_t ttl_ms = static_cast(ttl.count()); confirmed_deadline_boot_ms = attempt_start_boot_ms > std::numeric_limits::max() - ttl_ms ? std::numeric_limits::max() : attempt_start_boot_ms + ttl_ms; result.outcome = MountRenewOutcome::Committed; - result.attempts_sent = committed->attempts_sent; + result.attempts_sent = attempts_sent; result.resolved_by_read = committed->resolved_by_read; - result.sent_any = committed->attempts_sent != 0; + result.sent_any = attempts_sent != 0; return result; } if (const Conflict * conflict = std::get_if(&*written)) { result.sent_any = true; - result.attempts_sent = conflict->attempts_sent; + result.attempts_sent = attempts_sent; try { throwRenewConflict(conflict->seen); @@ -1665,7 +2062,7 @@ MountRenewResult MountLeaseRenewer::renewOn( if (const Refused * refused = std::get_if(&*written)) { result.sent_any = true; - result.attempts_sent = refused->attempts_sent; + result.attempts_sent = attempts_sent; markMountRenewTermination(MountRenewTerminalClassification::DeterministicFailure); result.failure = std::make_exception_ptr(Exception( refused->store_error, @@ -1675,13 +2072,18 @@ MountRenewResult MountLeaseRenewer::renewOn( if (const GaveUp * gave_up = std::get_if(&*written)) { - result.sent_any = gave_up->sent_any; - result.attempts_sent = gave_up->attempts_sent; + /// Across every engine call of this renewal, not only the call that gave up. + result.sent_any = attempts_sent != 0; + result.attempts_sent = attempts_sent; if (gave_up->why == GaveUp::Why::Deadline) result.deadline_source = gave_up->deadline_source; - /// Nothing was sent and the node was already stopping: the lease is exactly as it was, so this - /// is a renewal that never ran, not one that lost its authority. + /// Read after the refusal, because a stop that lands mid-renewal is what refused it. + const bool cancelled = environment.cancelled && environment.cancelled(); + + /// The call that gave up sent nothing, every earlier call of a series ended in a refusal or in a + /// read that found the slot as our last commit left it, and the node is stopping: the renewal + /// never ran and the farewell may still hand the slot back. if (gave_up->why == GaveUp::Why::FenceLost && !gave_up->sent_any && cancelled) { markMountRenewTermination(MountRenewTerminalClassification::Cancelled); @@ -1712,7 +2114,7 @@ MountRenewResult MountLeaseRenewer::renewOn( "observation: {})", key, terminalClassificationName(classification), - gave_up->sent_any ? "at least one" : "no", + result.sent_any ? "at least one" : "no", detail::renderObservation(gave_up->last_seen))); return terminalResult(std::move(result)); } @@ -1770,6 +2172,7 @@ void MountLeaseRenewer::terminate(CasOperation & op) { seq += 1; last_etag = std::move(committed->etag); + last_body = body; emitMountEvent( event_sink, CasEventType::MountRelease, srid, "farewell", nullptr, "graceful release -- lease stamped already-expired and watermark retired"); @@ -1809,42 +2212,40 @@ void MountLeaseRenewer::release() terminate(op); } -void sweepOwnMountStaging(IObjectStorage & object_storage, const String & mount_staging_prefix) noexcept +void sweepOwnMountStaging(IObjectStorage & object_storage, const String & mount_staging_prefix, uint64_t live_writer_epoch) noexcept { try { - /// max_keys=0 asks `listObjects` for the FULL listing under the prefix (it paginates until - /// exhausted rather than capping at some default page size) — see `IObjectStorage::listObjects`. - /// A mount's own staging debris is bounded (one mount's in-flight + leaked uploads), so a single - /// unbounded LIST at startup is acceptable; unlike GC's per-round budgets, this runs once per - /// mount, not on a recurring schedule. + /// `max_keys=0` lists the whole prefix. One mount's staging debris is bounded and this runs once per + /// mount, not on GC's recurring schedule, so an unbounded LIST is acceptable here. RelativePathsWithMetadata children; object_storage.listObjects(mount_staging_prefix, children, /*max_keys=*/0); - size_t removed = 0; for (const auto & child : children) { + const String & key = child->relative_path; + if (!key.starts_with(mount_staging_prefix)) + continue; + const std::optional epoch = Layout::stagedEpochOf(std::string_view(key).substr(mount_staging_prefix.size())); + if (!epoch || *epoch >= live_writer_epoch) + continue; /// the live epoch's own in-flight uploads, or a nested member's keys try { - object_storage.removeObjectIfExists(StoredObject(child->relative_path)); + object_storage.removeObjectIfExists(StoredObject(key)); ++removed; } catch (...) // NOLINT(bugprone-empty-catch) { - /// Best-effort: one stubborn key must not abort the sweep of the rest — it is retried - /// by a later mount's sweep. + /// One stubborn key must not abort the sweep; a later mount retries it. } } - if (removed) - LOG_INFO(getLogger("CasStagingSweeper"), - "Reclaimed {} leaked S3 staging object(s) under '{}' at mount start", - removed, mount_staging_prefix); + LOG_INFO(getLogger("CasStagingSweeper"), "Reclaimed {} staging object(s) below writer epoch {} under '{}' at mount start", + removed, live_writer_epoch, mount_staging_prefix); } catch (...) // NOLINT(bugprone-empty-catch) { - /// Best-effort: a LIST failure (a transient backend hiccup) at mount time must never fail the - /// mount — any leaked staging objects are bounded debris, reclaimed by a later mount's sweep. + /// A LIST failure at mount time must never fail the mount; the debris is bounded and retried later. } } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.h index f583ca2af83ca..63761f04f67b0 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Pool/CasServerRoot.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -67,21 +68,21 @@ struct MountRenewOperationEnvironment /// Facts the mount fence cannot see (park requested, pool no longer live, shutdown). FALSE ends /// the renewal exactly as a lost fence does; the engine does not need to know which refused. std::function live; - /// Sampled ONCE before the write. A renewal refused before its first attempt is reported as - /// `NotAttempted` rather than terminal only when this node had already been asked to stop -- - /// sampling it afterwards would read a flag that the refusal itself may have set. + /// Read once the write has given up. A renewal refused before its first attempt is reported as + /// `NotAttempted` rather than terminal when this node has been asked to stop by then. std::function cancelled; + RequestObserver requests; + /// The background renewal's wait after the store refused a request: the store said it cannot serve + /// it, and each engine call may refresh the credentials once, so a pace of one spacing would hammer + /// the store and refresh every second. + uint64_t refusal_pause_ms = 0; }; /// Validate a `server_root_id` — the explicit, configured identity of the content-addressed layout -/// subtree a server owns. It is a clean relative path: it composes into the -/// object-key tree (`gc/server-roots//...`, `roots//...`), so the same hygiene the layout -/// applies to a namespace applies here (mirrors `CasLayout.h::checkNamespace`): -/// - non-empty; -/// - no leading/trailing '/', no empty segment ("//"); -/// - no '.' or '..' segment; -/// - total length <= 255; -/// - no segment equal to the reserved "_files" / "_manifests". +/// subtree a server owns. It composes into object keys (`gc/server-roots//...`, `roots//...`) +/// as ONE path component: non-empty, at most 255 bytes, no '/', not '.' or '..', not the reserved +/// "_files" / "_manifests". A '/' is refused even for an id that already owns its owner object, because +/// decommission and the staging sweep select a member's objects by key prefix. /// Throws `ErrorCodes::BAD_ARGUMENTS` on any violation. Fail closed — there is no sanitizing fallback. inline void validateServerRootId(const String & id) { @@ -92,27 +93,22 @@ inline void validateServerRootId(const String & id) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "server_root_id '{}' is too long ({} > 255 bytes)", id, id.size()); - size_t start = 0; - while (true) - { - size_t end = id.find('/', start); - const String segment = id.substr(start, end == String::npos ? String::npos : end - start); - if (segment.empty()) - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, - "server_root_id '{}' has an empty segment (leading/trailing or doubled '/')", id); - if (segment == "." || segment == "..") - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, - "server_root_id '{}' uses a relative segment ('.' or '..')", id); - if (segment == "_files") - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, - "server_root_id '{}' uses the reserved segment '_files'", id); - if (segment == "_manifests") - throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, - "server_root_id '{}' uses the reserved segment '_manifests'", id); - if (end == String::npos) - break; - start = end + 1; - } + if (id.find('/') != String::npos) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "server_root_id '{}' contains '/'. A decommission selects a member's objects by key prefix, so a " + "member 'a/b' would be drained by a decommission of 'a'. Recreate the server that uses this id under a new " + " without '/'; the objects of the old id stay in the pool and are not visible " + "under the new id", + id); + if (id == "." || id == "..") + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "server_root_id '{}' uses a relative segment ('.' or '..')", id); + if (id == "_files") + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "server_root_id '{}' uses the reserved segment '_files'", id); + if (id == "_manifests") + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, + "server_root_id '{}' uses the reserved segment '_manifests'", id); } /// The per-server-root control objects (owner, epoch, and mount lease) and their text codecs are kept @@ -132,6 +128,18 @@ class Layout; bool serverRootSubtreeEmpty( CasOperation & op, const Layout & l, const String & srid, const RefCatalog & catalog_observation); +/// The first part of a server root's subtree that is not empty, checked in this order. +enum class ServerRootLeftover +{ + None, + CatalogRows, + Manifests, + RootsObjects, +}; + +ServerRootLeftover serverRootLeftover( + CasOperation & op, const Layout & l, const String & srid, const RefCatalog & catalog_observation); + /// Supplied by the pool layer so the low-level server-root protocol always observes the mandatory /// catalog. Every absent-control retry obtains a fresh, successfully decoded observation. using ObserveRefCatalog = std::function; @@ -142,27 +150,56 @@ using ObserveRefCatalog = std::function; /// counterpart used by normal opens and reuses this GET+decode path. std::optional readOwnerUuid(CasOperation & op, const Layout & l, const String & server_root_id); -/// Claim (or validate) the sticky owner anchor that binds `srid` to a server UUID (identity). -/// - owner present, equal `our_uuid`, and not tombstoned → ok (return); -/// - owner present and tombstoned → throw `CORRUPTED_DATA` (explicitly retired — fail closed); -/// - owner present, different → throw `CORRUPTED_DATA` (foreign owner — fail closed); -/// - owner absent AND the subtree is provably empty → `create` the owner (claim); -/// - owner absent BUT the subtree is non-empty → throw `CORRUPTED_DATA` (identity lost over -/// existing data — never silently re-claim). -/// The owner object is never deleted and never reassigned to a different UUID. -void claimOwnerOrThrow( +enum class RetiredOwnerPolicy : uint8_t +{ + Refuse, /// a remount, and every check after the owner claim, never accept a tombstoned owner + AcceptForDecommission, /// the decommission admin impersonates a retired victim to drain or rerun + RejoinIfVerified, /// a server start takes over a verified, drained retirement as an empty member +}; + +/// Claim or validate the owner anchor that binds `srid` to a server uuid, and return the owner life. +/// - present, equal `our_uuid`, not tombstoned → the stored life; +/// - present and tombstoned, under `RejoinIfVerified` (a server start), any non-zero uuid → rejoin as an +/// empty member: the owner becomes `{our_uuid, owner_life + 1}` and that life is returned. Refused with +/// `CORRUPTED_DATA` (zero uuid, `epoch` or `mount` missing, retirement not verified) or `ABORTED` (slot held, +/// data of the old life left, owner or slot changing on every decision); +/// - present, equal `our_uuid`, tombstoned → `CORRUPTED_DATA` under `Refuse`; the stored life under +/// `AcceptForDecommission`; +/// - present, different uuid, outside a rejoin → `CORRUPTED_DATA` (foreign owner), tombstoned or not; +/// - absent, subtree provably empty → create `{our_uuid, owner_life = 1}` and return 1; +/// - absent, subtree non-empty → `CORRUPTED_DATA` (identity lost over data). +/// The returned life is 0 for a body written before the field existed. +/// The owner object is never deleted; a rejoin is the one supported change of its uuid. +/// `sink` receives one `MountClaim` event per rejoin write. +uint64_t claimOwnerOrThrow( CasOperation & op, const Layout & l, const String & srid, UInt128 our_uuid, - const ObserveRefCatalog & observe_catalog); + const ObserveRefCatalog & observe_catalog, RetiredOwnerPolicy retired_policy = RetiredOwnerPolicy::Refuse, + const CasEventSink & sink = {}); + +/// Why a self-remount can never resume under `(our_uuid, recorded_life)`: the owner belongs to another server, is +/// retired, or is at another life. Empty when the owner allows it or is absent; the claim decides the absent case. +/// A tombstone written by 26.6.4 drops `owner_life` and reads as life 0, so the retirement is checked on its own. +String remountOwnerRefusal(CasOperation & op, const Layout & l, const String & srid, UInt128 our_uuid, uint64_t recorded_life); -/// Which mint policy governs `allocateWriterEpoch`'s absent-epoch branch (see below). -enum class EpochMintPolicy : uint8_t +/// What a claimer requires of the owner: after its body is in the slot and before it installs the renewer, +/// and, for a foreign terminal body, before the reclaim write. +struct OwnerExpectation { - NormalMount, /// absent-epoch re-mint requires authoritative mount absence - DecommissionRecovery, /// absent-epoch re-mint requires a TERMINAL mount; mints a distinct epoch + UInt128 server_uuid; + uint64_t owner_life; + RetiredOwnerPolicy retired; }; +/// One `GET owner` after `claimMount` returned `Claimed`. Throws `CORRUPTED_DATA` unless the owner is exactly +/// `(expected.server_uuid, expected.owner_life)` and, under `Refuse`, not retired. Before throwing it rewrites +/// the claimer's own `(uuid, our_epoch)` body `gc_fenced`, so a successor need not wait the lease out; a +/// failed fence write is logged and the refusal is still thrown. +void recheckOwnerAfterClaim( + CasOperation & op, const Layout & l, const String & srid, const OwnerExpectation & expected, + uint64_t our_epoch, const CasEventSink & sink = {}); + /// Allocate the next durable-monotone `writer_epoch` by CAS-bumping the sticky `epoch` object -/// (`ServerEpoch{next_writer_epoch}`), returning the value the caller adopts as its writer_epoch. +/// (`ServerEpoch{next_writer_epoch, allocation_nonce}`), returning the value the caller adopts as its writer_epoch. /// - `epoch` absent AND the subtree is non-empty → throw `CORRUPTED_DATA` (missing epoch over /// data is a reset hazard); /// - `epoch` absent AND the subtree is empty → the absent-epoch branch is a LIFECYCLE decision, @@ -170,24 +207,21 @@ enum class EpochMintPolicy : uint8_t /// flattens transport faults into "not found"), to decide whether the mount object is really /// gone: /// - `KeyAbsent` → authoritative absence, mint epoch 1 (fresh-root bootstrap); -/// - `Present`, `policy == NormalMount` → throw `CORRUPTED_DATA` (durable epoch state was -/// lost while a mount is live or recently live — refusing to re-mint epoch 1 there is how a -/// same-(uuid, epoch) twin is avoided); -/// - `Present`, `policy == DecommissionRecovery` → the surviving mount must be TERMINAL (not -/// live); a live member throws `ABORTED`, otherwise mint `surviving.writer_epoch + 1` — -/// distinct from the survivor's by construction (`now_ms` is required, nonzero, here); +/// - `Present` → throw `CORRUPTED_DATA` (durable epoch state was lost while a mount is live or +/// recently live; refusing to re-mint epoch 1 is how a same-(uuid, epoch) twin is avoided. If no +/// server uses this root, start with a new server_root_id; a decommission never recreates a counter); /// - anything else (`ContainerAbsent`/`AccessDenied`/`Indeterminate`) → throw /// `CORRUPTED_DATA` (absence was never proven; fail closed); /// - otherwise read `next = current.next_writer_epoch`, write `{next + 1}` against the observed /// incarnation, re-deciding on conflict, and return `next`. +/// Every decision writes a fresh `allocation_nonce`, so an unclear write counts as ours only when the +/// stored bytes are exactly ours; a same-uuid twin bumping to the same value is a conflict. uint64_t allocateWriterEpoch(CasOperation & op, const Layout & l, const String & srid, - EpochMintPolicy policy, uint64_t now_ms, const ObserveRefCatalog & observe_catalog); /// Which certificate of death, or the operator's explicit unsafe authorization, justified a -/// same-uuid, different-epoch mount reclaim. `None` when no -/// reclaim of that kind happened (a fresh claim, a -/// same-epoch refresh, `LiveDoubleStart`, `ForeignOwner`, `FencedSelf`). +/// older-epoch mount reclaim (`Fenced` and `Clean` also cover another uuid's terminal body). `None` when no +/// reclaim of that kind happened (a fresh claim, `LiveDoubleStart`, `ForeignOwner`, `FencedSelf`). enum class MountPriorState { None, @@ -202,12 +236,13 @@ enum class MountPriorState /// already settled who may write; the lease settles whether a live incarnation currently holds the /// slot. Decision over `get(mountKey)`: /// - absent → write our body via `create` → `Claimed`; -/// - same `server_uuid` AND same `writer_epoch` as (our_uuid, our_epoch) → it is OUR OWN claim -/// (a replay / the renewer adopting it): -/// - `gc_fenced` → terminal for THIS (uuid, epoch) — a fence costs an epoch, so refreshing it -/// in place would reactivate a fenced incarnation → `FencedSelf` (no write); -/// - otherwise → refresh (`replace` to bump seq + fresh `expires_at_ms`) → `Claimed`; -/// - same `server_uuid`, DIFFERENT `writer_epoch` → reclaimed ONLY on a certificate of death, or the +/// - same `server_uuid` and a NEWER `writer_epoch` than `our_epoch` → throw `ABORTED`, no write: an +/// older allocation never replaces a newer incarnation, whatever certifies its end; +/// - same `server_uuid` AND same `writer_epoch` → another process was allocated our epoch; never +/// refreshed: +/// - `gc_fenced` → terminal for THIS (uuid, epoch) → `FencedSelf` (no write); +/// - otherwise → `LiveDoubleStart` (no write); +/// - same `server_uuid`, OLDER `writer_epoch` → reclaimed ONLY on a certificate of death, or the /// operator's explicit unsafe authorization, that /// needs no fresh wall-clock trust (see /// `claimMountAwaitingExpiry` below for how a plain "looks expired" reading is turned into one): @@ -225,7 +260,10 @@ enum class MountPriorState /// now_ms` ALONE is never sufficient — comparing a predecessor's stamp against OUR wall clock /// is unsafe because a clock-skewed or merely late-observing /// caller must never conclude death from a bare timestamp read); -/// - different `server_uuid` → `ForeignOwner` (do NOT write, regardless of expiry or prior state). +/// - different `server_uuid` → `ForeignOwner`, no write, unless the body is terminal (`gc_fenced` or the +/// clean marker), its `writer_epoch` is below `our_epoch`, and a fresh owner read matches +/// `foreign_reclaim_owner`: then it is reclaimed like an older epoch of our own (`prior = Fenced` or +/// `Clean`). A rejoin under a new uuid leaves such a body. struct MountClaimResult { /// Plain (unscoped) enum: callers compare with `MountClaimResult::Claimed` directly. @@ -248,8 +286,8 @@ struct MountClaimResult /// rather than the proposal, because a caller cannot check a convention it cannot see. std::optional body; /// Which certificate of death, or the operator's explicit unsafe authorization, justified a - /// same-uuid, different-epoch `Claimed` reclaim (`None` for - /// every other `Kind`, and for the absent-slot / same-epoch-refresh `Claimed` cases). + /// same-uuid, older-epoch `Claimed` reclaim (`None` for + /// every other `Kind`, and for the absent-slot `Claimed` case). MountPriorState prior = MountPriorState::None; /// The incarnation of the body this result observed, for /// `LiveDoubleStart` only (a fresh `Claimed`/`FencedSelf`/`ForeignOwner` write/observe has no @@ -271,8 +309,14 @@ class MountFencedException : public DB::Exception public: explicit MountFencedException(const String & msg) : DB::Exception(msg, DB::ErrorCodes::ABORTED) {} + + /// Copies of this exception (`copyMutableException`) must keep the type that catch sites match. + MountFencedException * clone() const override { return new MountFencedException(*this); } + void rethrow() const override { throw *this; } // NOLINT }; +/// A caller never claims twice with one `writer_epoch`; a retry allocates a new one. That is why a +/// body of our own `(uuid, epoch)` found in the slot belongs to another process. /// `proven_dead_incarnation`: the incarnation of a same-uuid, different-epoch lease that the CALLER /// already proved dead by observation (see `claimMountAwaitingExpiry`) — matching it against the /// CURRENTLY observed incarnation is the ONLY way (besides `gc_fenced` / the clean marker) a @@ -285,7 +329,8 @@ class MountFencedException : public DB::Exception MountClaimResult claimMount( CasOperation & op, const Layout & l, const String & srid, UInt128 our_uuid, uint64_t our_epoch, uint64_t now_ms, uint64_t ttl_ms, const std::optional & proven_dead_incarnation = {}, - const CasEventSink & sink = {}, const std::optional & unsafe_reclaim_authorization = {}); + const CasEventSink & sink = {}, const std::optional & unsafe_reclaim_authorization = {}, + const std::optional & foreign_reclaim_owner = {}); /// Format the operator-actionable startup error shown when the mount lease is held by a genuinely /// live second server (the same `server_root_id` is mounted twice). Produced only AFTER this server @@ -298,15 +343,17 @@ String mountDoubleStartMessage(const String & srid, const std::optional & sleep_ms_fn, const std::function & on_wait_start = {}, - const CasEventSink & sink = {}); + const CasEventSink & sink = {}, + const std::optional & foreign_reclaim_owner = {}); /// One `server_root_id`'s cross-round incarnation-stability observation, /// owned by the GC leader instance (`Cas::Gc::mount_obs`) and threaded through consecutive @@ -379,7 +429,8 @@ using MountObservationMap = std::map; /// CURRENT incarnation differs from (or is absent from) `obs` is (re)started fresh — counted `live`, /// never fenced this call, regardless of what its stamped `expires_at_ms` claims (a bare /// wall-clock stamp is never trusted — see `claimMount`'s "certificate of death" doc). Only once -/// the SAME incarnation has held for `>= stable_threshold_ms` OF THE LEADER'S OWN CLOCK does the body +/// the SAME incarnation has held, on the leader's own clock, for +/// `mountObservationThresholdMs(observer_lease_ms, declaredMountLeaseMs(body), cadence_ms)` does the body /// become FENCE-eligible; /// - FENCE-eligible → one incarnation-guarded `replace` preserving the WHOLE body, setting /// `gc_fenced = true` and `seq + 1`. On `Committed` → excluded (`fenced_now`); on `Conflict` @@ -394,6 +445,9 @@ using MountObservationMap = std::map; /// each sighting. `obs` is owned by the caller and threaded across consecutive calls (one GC /// leader instance, `Cas::Gc::mount_obs`) — a fresh leader starts with an empty map (safe: delays /// fencing one round, never fences early). +/// A body whose declared lease is non-zero and differs from `observer_lease_ms` is named in one `WARNING` +/// per srid; `lease_mismatch_warned` holds the srids already named. The check never changes a classification +/// or a write. `obs` and `lease_mismatch_warned` drop every srid this pass did not list. /// /// The fence-out is BOTH safety and liveness. Safety: a sleeper's later renewal permanently fails /// (its `replace` now mismatches the fenced incarnation → `tripMountLost`), so it can never re-arm @@ -412,7 +466,8 @@ struct HeartbeatFloor HeartbeatFloor computeHeartbeatFloor(CasOperation & op, const Layout & l, uint64_t now_ms, const std::function & mono_ms_fn, - uint64_t stable_threshold_ms, MountObservationMap & obs); + uint64_t observer_lease_ms, uint64_t cadence_ms, MountObservationMap & obs, + std::set & lease_mismatch_warned); /// One `gc/server-roots//mount` slot whose holder is not provably finished with the prefix. struct NonTerminalMountSlot @@ -443,16 +498,20 @@ std::vector probeNonTerminalMountSlots(CasOperation & op, /// A read-only snapshot of one server's mount slot, for introspection (`system.cas_mounts`). /// state: `live` (lease within TTL+skew), `expired` (lease ran out; the next GC round's heartbeat floor /// will fence it), `terminated` (clean farewell: `min_active_build_sequence == UINT64_MAX`), `fenced` (`gc_fenced`), +/// `retired` (terminated or fenced, owner tombstoned; one more `GET` per terminated or fenced row), /// `corrupt` (body failed to decode — surfaced as a row, never an exception). struct MountInfo { String srid; MountLease lease; String state; + /// For a `retired` row: whether a decommission verified the retirement. A rerun is for an unverified one only. + bool retirement_verified = false; }; /// Enumerate every mount slot under `gc/server-roots/`, decoded and classified — the read-only sibling -/// of `computeHeartbeatFloor`: ZERO writes (no fence-out), per-row fail-open. One LIST + one GET per slot. +/// of `computeHeartbeatFloor`: ZERO writes (no fence-out); an undecodable mount body, or an undecodable owner of a +/// terminal row, is a `corrupt` row. One LIST, one GET per slot, and one more per terminated or fenced slot. std::vector listMounts(CasOperation & op, const Layout & layout, uint64_t now_ms, uint64_t skew_margin_ms); /// Whether the mounted writer identified by `(server_root_id, writer_epoch)` (the two fields of a @@ -526,12 +585,13 @@ bool isCreatorFenceTerminal(CasOperation & op, const Layout & layout, const Stri /// - foreign uuid → fail closed; /// - absent → `create`; expired-our-uuid (any epoch) → `replace` reclaim. /// -/// PLANES. Only the RENEWAL is admitted under the mount fence, because only a renewal writes under -/// authority the fence is tracking. The claim and the farewell are admitted off it: a self-remount -/// claims with the fence already latched lost, so a claim gated on the fence could never reclaim, and -/// a farewell refused because the fence has run down would leave the slot looking live until GC -/// fences it out. Neither is unguarded: a claim's safety is its own conditional write, and a caller -/// that has shutdown facts hands them over as a `Liveness`. +/// PLANES. The bounded renewal (`renew`: startup and direct calls) is admitted under the mount fence, +/// because it writes under the authority the fence tracks. The background renewal, the claim, the +/// remount re-anchor and the farewell run on the lease plane, whose fence is open: the background +/// renewal must outlive its lease, a self-remount claims with the fence already latched lost, and a +/// farewell refused because the fence ran down would leave the slot looking live until GC fences it +/// out. None is unguarded: every write is conditional, and a caller with shutdown facts hands them +/// over as a `Liveness`. class MountLeaseRenewer { public: @@ -549,7 +609,7 @@ class MountLeaseRenewer /// Adopt the already-claimed mount. Returns the exact pre-I/O BOOTTIME anchor. `liveness` carries /// the caller's shutdown terms; the mount fence is deliberately not consulted here. uint64_t start(Liveness liveness = {}); - /// The steady-state renewal, admitted under the mount fence. + /// The bounded renewal, admitted under the mount fence: the startup re-anchor and direct calls. MountRenewResult renew(const MountRenewOperationEnvironment & environment); /// The remount's re-anchor, which is bootstrap control rather than steady state: a remount renews /// BEFORE it arms the fence for the new incarnation, so the fence is still latched lost and an @@ -557,6 +617,10 @@ class MountLeaseRenewer /// renewer's own open plane, the one the claim and the farewell use, so there is no plane for a /// caller to get wrong. Same policy and same verdicts as `renew`. MountRenewResult renewForRemount(const MountRenewOperationEnvironment & environment = {}); + /// The background renewal, on the lease plane, retried until a definitive answer: a commit, a + /// conflict the read observed, a local failure, or `environment.live` refusing. A refusal sends the + /// same body again, and a refused precondition whose read failed repeats only the read. + MountRenewResult renewUntilDefinitive(const MountRenewOperationEnvironment & environment); void release(); MountLeaseRenewerState state() const { return renewer_state; } @@ -568,8 +632,14 @@ class MountLeaseRenewer /// The incarnation every guarded write of this slot names. Engaged for exactly the states that /// admit such a write: `start` establishes it and each committed renewal replaces it. const Etag & precondition() const; - /// One renewal admitted on `plane`; `renew` and `renewForRemount` differ only in which they pass. - MountRenewResult renewOn(CasRequests & plane, const MountRenewOperationEnvironment & environment); + /// One renewal admitted on `plane`. `renew` and `renewForRemount` pass `until_definitive = false` (bounded by + /// the lease); `renewUntilDefinitive` passes `true` (the background renewal's unbounded retry). + MountRenewResult renewOn(CasRequests & plane, const MountRenewOperationEnvironment & environment, bool until_definitive); + /// Repeats the read of a slot whose precondition was refused until it observes the slot: one spacing + /// after the write attempt that started at `write_started_boot_ms`, then `refusal_pause_ms` after each + /// refused read. `NotObserved` only when `op` is no longer admitted. + Observation readUntilObserved(CasOperation & op, const Retry & policy, + const MountRenewOperationEnvironment & environment, uint64_t write_started_boot_ms) const; Etag claim(CasOperation & op, const String & body); [[noreturn]] void throwRenewConflict(const Observation & seen) const; MountRenewResult terminalResult(MountRenewResult result); @@ -595,38 +665,26 @@ class MountLeaseRenewer /// The incarnation our last landed write created; every renewal and the farewell name it as the /// precondition. Unset only before `start` has landed one. std::optional last_etag; + /// The bytes `last_etag` names. A read that returns them under that token says the slot did not move. + String last_body; uint64_t confirmed_deadline_boot_ms = 0; uint64_t last_committed_attempt_start_boot_ms = 0; }; /// Mount-lease-scoped staging sweeper for objects left behind by S3-native staging. /// -/// A leaked S3 staging object happens two ways: (1) an exception between blob publication succeeding and -/// `cleanupPendingTempFiles` deleting the staging key (`ContentAddressedTransaction.cpp`), or (2) an -/// aborted/cancelled transaction whose pending blobs were staged but never promoted — by design, -/// `cleanupPendingTempFiles` deliberately leaves an S3 staging object in place on the abort path (never a -/// bare `fs::remove` on a remote key), so this sweeper is its ONLY reclaimer. Debris from either case is -/// bounded to `staging//` — the ONE mount that could ever have written under that prefix, since -/// every staging key this mount ever mints comes from `ContentAddressedMetadataStorage::stagingKeyPrefix()` -/// (`physicalKey(pool_prefix + "/staging/" + server_root_id)`), keyed by THIS mount's own `server_root_id`. -/// -/// LEASE-FENCE (fail-closed, never fail-open): `sweepOwnMountStaging` removes ONLY objects whose key -/// starts with the given `mount_staging_prefix` — pass your OWN mount's prefix, never another mount's. -/// The caller (`ContentAddressedMetadataStorage::startup()`) invokes this exactly once, at mount start, -/// with `stagingKeyPrefix() + "/"` — the SAME prefix construction the writer uses to mint staging keys, so -/// this sweep can never reach a different mount's `staging//` subtree: no other writer -/// ever stages a key under THIS mount's own `server_root_id` prefix, and this function never lists or -/// touches anything outside the prefix it is given. +/// Staging keys are `staging///.tmp`, minted from +/// `ContentAddressedMetadataStorage::stagingKeyPrefix()`. A key leaks when its transaction never ran +/// `cleanupPendingTempFiles` (the process died, or an exception hit between `sink->finalize()` and `on_finalized` +/// registering the key in `CaContentWriteBuffer::finalizeImpl`) or when its best-effort delete on commit or abort +/// was lost. While the member lives, this sweep is the only reclaimer. GC never lists `staging/`. /// -/// Best-effort and NEVER THROWS: one stubborn key (or a LIST failure) must never abort the sweep of the -/// rest, and must never fail the mount (mirrors `feedback_ca_gc_never_throw_on_404` — a throw here would -/// only wedge startup, not GC, but the same fail-open-on-error discipline applies to any best-effort -/// reclaim of debris). +/// Deletes `/` with `epoch < live_writer_epoch`, and flat +/// `` keys (epoch 0). The cutoff is the live epoch fixed at the start of the +/// sweep, so a successor's in-flight uploads (its epoch is not below the cutoff) survive. Every other path +/// is left alone, including another member's keys nested under this prefix. Pass your OWN mount's prefix. /// -/// GC excludes `staging/` entirely: GC blob discovery LISTs `Layout::blobsPrefix()` -/// (`/blobs/`) — a distinct top-level prefix from `staging/`, `cas/ns/`, and -/// `cas/manifests/` (see `CasLayout.h`) — so a `staging/` object is never listed, HEAD'd, or condemned by -/// GC's fold. This sweeper is the ONLY reclaimer of `staging/` debris. -void sweepOwnMountStaging(IObjectStorage & object_storage, const String & mount_staging_prefix) noexcept; +/// Best-effort and never throws: a LIST failure or one stubborn key must not fail the mount. +void sweepOwnMountStaging(IObjectStorage & object_storage, const String & mount_staging_prefix, uint64_t live_writer_epoch) noexcept; } diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.h index a67ea2f14b23e..a408df841e747 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Primitives/CasEvent.h @@ -32,10 +32,9 @@ enum class CasEventType GateRevalidate, GateResurrect, WatermarkRenew, MountRemount, MountClaim, MountRelease, MountConflict, /// Operator-driven erasure of a dead pool member's namespace. `decommissionPoolMember` runs as - /// a writer, never as GC: it claims the member's mount, drains its namespaces and debris, and - /// deletes the mount slot only after the drain is confirmed. The slot is the interrupted-operation - /// resume anchor, so a failed drain leaves it terminated for a later retry. `outcome` is one of - /// "begin", "namespace_removed", or "end". + /// a writer, never as GC: it claims the member's mount, tombstones its owner, drains its namespaces + /// and debris, and keeps the owner, epoch and mount objects. `outcome` is one of "begin", + /// "namespace_removed", or "end". MemberDecommission, /// Incidental-detection reaction to foreign bytes at a ref-log wedge key owned by this mount. /// The mount lease makes the key exclusive, so this is impossible under legitimate diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md index 44430e021ddbb..3c4ffaf4274b2 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md @@ -53,9 +53,9 @@ Everything persisted is a text format with a versioned header line; see - **Write**: `ContentAddressedTransaction` buffers or spills file writes (`scratch_path`, or opt-in S3 staging), hashes the payload, and hands it to - `Pool/CasPartWriteTxn`: start every materialization with blob `HEAD`, adopt a present - non-condemned body or publish the writer's source unconditionally, reconcile `.meta` to - `Clean`, and record explicit `Materialized` proof; then stage the manifest, `precommitAdd` the ref, + `Pool/CasPartWriteTxn`: start every materialization with blob `HEAD`, adopt only on a `Clean` + marker read after the `HEAD`, otherwise publish the writer's source unconditionally and reconcile + `.meta` to `Clean`, and record explicit `Materialized` proof; then stage the manifest, `precommitAdd` the ref, then promote it to committed in the ref log. - **Read**: disk path → `Parts/PartPathParser` (namespace + part + file) → ref table resolves the manifest → `Pool/CasManifestReader` locates the diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.cpp b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.cpp index 11a9ba3d7f94d..b20fc7be08644 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.cpp +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.cpp @@ -11,9 +11,9 @@ #include #include #include +#include #include #include -#include #include #include @@ -21,7 +21,9 @@ namespace DB { namespace ErrorCodes { + extern const int ABORTED; extern const int CORRUPTED_DATA; + extern const int LOGICAL_ERROR; } } @@ -47,11 +49,13 @@ std::string_view removalName(Removal r) /// fenced by the decommission claim (`Pool::openForDecommission`), so nothing should be racing these /// deletes, and a plain exact-incarnation delete of every listed object is race-free. /// +/// Runs on the admin's mount plane, so a fenced admin's deletes are refused locally; only attempts already +/// sent can land late. +/// /// A per-object failure — a backend exception, a `Mismatch` or `Gone` outcome, or an object /// disappearing between `LIST` and `HEAD` — is recorded as a warning and does not prevent the remaining -/// objects from being attempted. The caller keeps the pool slot whenever warnings are present, so the -/// terminated slot remains available as a resume anchor instead of being deleted after an unconfirmed -/// drain. Returns only the objects whose exact-incarnation delete was reported as `Removed`. +/// objects from being attempted while `op` is still admitted. Returns only the objects whose +/// exact-incarnation delete was reported as `Removed`. uint64_t deleteListedPrefix(CasOperation & op, const String & prefix, std::vector & warnings) { uint64_t deleted = 0; @@ -82,116 +86,262 @@ uint64_t deleteListedPrefix(CasOperation & op, const String & prefix, std::vecto { warnings.push_back("decommission drain: " + listed.key + " delete failed: " + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + return op.admitted(); } return true; }, Retry::standard()); return deleted; } -/// Delete one slot control object by an incarnation captured at the protocol-defined fence point. Slot -/// retirement is fail-closed: unlike the debris drains above, any non-`Removed` outcome or exception -/// stops the tail before it can touch the next control object. -bool deleteSlotObject(CasOperation & op, const String & key, const Etag & etag, std::vector & warnings) +String whyNotWritten(const WriteResult & result) { - try + return std::visit(detail::Overload{ + [](const Committed &) -> String { return "committed"; }, + [](const Declined &) -> String { return "declined"; }, + [](const Conflict &) -> String { return "the object changed since it was read"; }, + [](const Refused & refused) -> String { return "the store refused it: " + refused.message; }, + [](const GaveUp & gave_up) -> String + { + return gave_up.sent_any ? "gave up after sending, it may have landed" : "gave up before sending"; + }}, result); +} + +/// A mark counts only on a retired owner: clearing `retired_at_ms` by hand to restart a member leaves the mark +/// of the earlier life behind. +bool holdsVerifiedRetirement(const OwnerObject & owner) +{ + return owner.retirement_verified && owner.retired_at_ms.has_value(); +} + +/// This run's tombstone: what a later exit needs to decide whether to keep it or roll it back. +struct Tombstone +{ + OwnerObject owner; /// the tombstone's body + bool previously_retired = false; /// the owner was already retired; a rollback never undoes that + Etag etag; /// the tombstone's incarnation, the rollback's precondition +}; + +void logWarnings(const String & srid, const std::vector & warnings) +{ + for (const String & warning : warnings) + LOG_WARNING(getLogger("CasDecommission"), "CAS decommission '{}': {}", srid, warning); +} + +/// Undo this run's tombstone after a claim lost before the drain: the next life, not retired, no mark. An earlier retirement +/// stays: its tombstone landed while its run held the claim, so no member runs under it. A rollback whose +/// precondition moved was overtaken by a later owner write, which then owns the outcome. +void rollBackTombstone(CasOperation & op, const Layout & layout, const String & srid, const Tombstone & tombstone, + std::vector & warnings) +{ + if (tombstone.previously_retired) { - const Removal outcome = op.remove(key, etag, Retry::standard()); - if (outcome == Removal::Removed) - return true; + warnings.push_back("the admin claim was lost after the tombstone; the owner keeps its earlier retirement"); + return; + } + const OwnerObject restored = nextOwnerLife(tombstone.owner, std::nullopt, false); + const WriteResult written = op.replace(layout.ownerKey(srid), encodeOwner(restored), tombstone.etag, Retry::standard()); + if (std::holds_alternative(written)) + warnings.push_back("the admin claim was lost after the tombstone; the tombstone was rolled back (owner life " + + std::to_string(restored.owner_life) + "); rerun to decommission"); + else if (std::holds_alternative(written)) + warnings.push_back("the admin claim was lost after the tombstone and the rollback did not land: the owner " + "changed after this run's tombstone; read the owner object '" + layout.ownerKey(srid) + + "' before rerunning"); + else + warnings.push_back("the admin claim was lost after the tombstone and the rollback did not complete (" + + whyNotWritten(written) + "); read the owner object '" + layout.ownerKey(srid) + + "' before rerunning"); +} - warnings.push_back("slot delete failed: " + key + ": delete outcome " - + String(removalName(outcome))); +/// Every exit after the tombstone and before the farewell: the tombstone stays while the claim holds. A lost claim +/// rolls it back only before the drain starts: once the drain may have removed data, the tombstone is what keeps the +/// member from mounting over the hole, and a rerun finishes the retirement. A failed confirm is a warning, so an exit +/// that already carries an error still propagates its own. +void confirmClaimOrRollBack(CasOperation & op, const Layout & layout, const String & srid, UInt128 victim_uuid, + uint64_t admin_epoch, const Tombstone & tombstone, bool drain_started, + std::vector & warnings) +{ + try + { + if (adminClaimStillHeld(op, layout, srid, victim_uuid, admin_epoch)) + return; + if (!drain_started) + rollBackTombstone(op, layout, srid, tombstone, warnings); + else + warnings.push_back("the admin claim was lost during the drain; the tombstone stays because the drain may " + "have removed data; do not start this member, rerun SYSTEM CAS DROP POOL MEMBER"); } catch (...) { - warnings.push_back("slot delete failed: " + key + ": " - + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + warnings.push_back("could not confirm the admin claim: " + getCurrentExceptionMessage(/*with_stacktrace=*/false) + + "; read the owner object '" + layout.ownerKey(srid) + "' before rerunning"); } - return false; -} - } -DecommissionReport decommissionPoolMember(BackendPtr backend, PoolConfig config, - const String & victim_srid, const CasEventSink & sink, - const std::function & request_gc_round, - const std::function & drain_now_fn, - const std::function & drain_sleep_fn) +/// Reads the owner on `control`, requires the victim's uuid, and replaces it on the admin's mount plane by the +/// next life, retired, keeping an earlier verified mark. The mount plane sends no attempt after the admin's +/// lease deadline, but the store may apply one later. A paused earlier run cannot roll this tombstone back: +/// its precondition is older. A write with an unknown outcome is still an exit after the tombstone when it +/// landed: if the owner reads back as exactly these bytes, the claim is confirmed (or the tombstone rolled +/// back) before the error propagates. Otherwise the admin withholds its clean farewell: the victim then runs +/// only after its slot is fenced or observed dead. +Tombstone writeTombstone(CasOperation & control, Pool & admin, const Layout & layout, const String & srid, + UInt128 victim_uuid, uint64_t admin_epoch, std::vector & warnings) { - DecommissionReport report; - report.srid = victim_srid; - bool gc_round_needed = false; - /// A namespace may have reached `Removing` before a later namespace fails closed. Preserve the - /// already-earned liveness signal on every exit: the callback only wakes the existing serialized - /// GC worker and cannot perform catalog work itself. - SCOPE_EXIT({ - if (gc_round_needed && request_gc_round) - request_gc_round(); - }); - - /// Decommission is administrative and non-hot-path: it has no mount-lease fence of its own -- the - /// exact-incarnation compare on every write is the safety mechanism -- so it opens its own - /// always-admitted request engine rather than one of `Pool`'s fenced planes. The pre-impersonation - /// cut below runs against the raw backend passed in, because `Pool::openForDecommission` has not - /// yet wrapped it for instrumentation and no `Pool` exists yet to route through. - CasRequests preflight_requests(backend, Fence::open()); - CasOperation preflight_op = preflight_requests.admit(); - - /// Validate one required immutable ownership cut before impersonating the victim. The admin open - /// performs its own fresh catalog observation for mount safety, but namespace selection below - /// must reuse this exact pre-mutation decision rather than read a later authority set. - const Layout catalog_layout(config.pool_prefix); - const CasRefCatalog::Snapshot catalog_cut = CasRefCatalog::read(preflight_op, catalog_layout); - catalog_cut.life_index.throwIfAmbiguous("CAS decommission"); - - /// `drain_now_fn`/`drain_sleep_fn` below replace the clock and sleep the STANDALONE `requests` - /// engine (opened further down) paces its own retries on, but `config.boot_ms_fn`/ - /// `config.retry_sleep_fn` -- the seams `Pool::openForDecommission` itself constructs its - /// mount/farewell/GC planes with -- are distinct. Left unset, those planes retry on the real boot - /// clock and a real sleep, and a caller that fakes only the standalone engine's clock ends up - /// comparing it against an unrelated one at the mount lease's farewell bound (or, worse, against a - /// clock that only the standalone engine's sleep advances: a retry loop on `mount_requests`/ - /// `gc_requests` bound to that frozen clock while actually sleeping for real would never see its - /// own deadline elapse). Fold BOTH into `config` here, together, unless the caller already asked - /// for a specific clock or sleep of its own -- installing only one of the two is exactly the - /// half-fix that leaves the other seam retrying forever. - if (drain_now_fn && drain_sleep_fn) + CasOperation fenced = admin.mountRequests().admit(); + const String key = layout.ownerKey(srid); + const std::optional current = control.read(key, Retry::standard()); + if (!current) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "CAS decommission '{}': the owner anchor vanished after the admin claim", srid); + const OwnerObject read = decodeOwner(current->bytes); + if (read.server_uuid != victim_uuid) + throw Exception(ErrorCodes::ABORTED, + "CAS decommission '{}': the owner is now server_uuid={}, the claim was made for {}; another server " + "took this server_root_id since the claim, nothing was removed", + srid, u128ToHex(read.server_uuid), u128ToHex(victim_uuid)); + + const bool previously_retired = read.retired_at_ms.has_value(); + const OwnerObject owner = nextOwnerLife(read, nowMs(), holdsVerifiedRetirement(read)); + const String bytes = encodeOwner(owner); + const String what = fmt::format("CAS decommission '{}' owner tombstone", srid); + + std::optional written; + std::exception_ptr unknown_outcome; + try { - if (!config.boot_ms_fn) - config.boot_ms_fn = drain_now_fn; - if (!config.retry_sleep_fn) - config.retry_sleep_fn = drain_sleep_fn; + written = fenced.replace(key, bytes, current->etag, Retry::standard()); + } + catch (...) + { + /// The engine rethrows a local failure even after an attempt was sent. + unknown_outcome = std::current_exception(); + } + if (written) + { + if (const Committed * committed = std::get_if(&*written)) + return Tombstone{.owner = owner, .previously_retired = previously_retired, .etag = committed->etag}; + if (std::holds_alternative(*written)) + throw Exception(ErrorCodes::ABORTED, + "CAS decommission '{}': the owner was rewritten between the claim and the tombstone (a rerun or a " + "rejoin); nothing was removed, rerun to decide against the current owner", srid); + const GaveUp * gave_up = std::get_if(&*written); + if (!gave_up || !gave_up->sent_any) + { + orThrow(std::move(*written), what); + throw Exception(ErrorCodes::LOGICAL_ERROR, "{}: a conditional replace was declined", what); + } + try + { + orThrow(std::move(*written), what); + } + catch (...) + { + unknown_outcome = std::current_exception(); + } } - config.event_sink = sink; - PoolPtr admin = Pool::openForDecommission(std::move(backend), std::move(config), victim_srid); - if (drain_now_fn && drain_sleep_fn) + std::optional landed; + try { - /// Re-affirms the same values `config` above already installed on `mount_requests`/ - /// `farewell_requests`/`gc_requests` at construction, and additionally wires `ref_ledger`'s own - /// retry sleep, which has no construction-time seam of its own. `sweepNamespace` below issues - /// its deletes on `admin`'s own GC plane. - admin->setCasRequestNowFnForTest(drain_now_fn); - admin->setCasRetrySleepForTest(drain_sleep_fn); + landed = control.read(key, Retry::standard()); } + catch (...) + { + warnings.push_back("could not read the owner back after the tombstone write: " + + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + } + if (landed && landed->bytes == bytes) + { + const Tombstone tombstone{.owner = owner, .previously_retired = previously_retired, .etag = landed->etag}; + confirmClaimOrRollBack(control, layout, srid, victim_uuid, admin_epoch, tombstone, /*drain_started=*/false, warnings); + } + else + admin.withholdFarewell(); + logWarnings(srid, warnings); + std::rethrow_exception(unknown_outcome); +} - /// A second engine over the pool's own (now instrumented) backend: `CasRequests` keeps its own - /// shared_ptr to it, so `op` stays usable after `admin.reset()` retires the `Pool` below. - CasRequests requests(admin->poolBackendPtr(), Fence::open()); - if (drain_now_fn && drain_sleep_fn) +/// Whether the owner carries a verified retirement once this run can no longer act. Otherwise says why not: +/// an unretired owner past this run's tombstone moved on (a rejoin, or another run's rollback), and a rerun +/// there could drain a rejoined member; anything else, a later run's unverified tombstone included, means +/// rerun. +bool ownerHoldsVerifiedRetirement(CasOperation & op, const Layout & layout, const String & srid, const Tombstone & tombstone, + std::vector & warnings) +{ + try { - requests.setNowFnForTest(drain_now_fn); - requests.setSleepFnForTest(drain_sleep_fn); + if (const std::optional current = op.read(layout.ownerKey(srid), Retry::standard())) + { + const OwnerObject owner = decodeOwner(current->bytes); + if (holdsVerifiedRetirement(owner)) + return true; + if (!owner.retired_at_ms && owner.owner_life > tombstone.owner.owner_life) + { + warnings.push_back(fmt::format( + "the owner moved on to life {} after this run's tombstone (a rejoin or another run's rollback); " + "read system.cas_mounts and the owner object '{}' before rerunning", + owner.owner_life, layout.ownerKey(srid))); + return false; + } + } + } + catch (...) + { + warnings.push_back("could not re-read the owner: " + getCurrentExceptionMessage(/*with_stacktrace=*/false)); } - CasOperation op = requests.admit(); + warnings.push_back("the retirement is not verified; rerun SYSTEM CAS DROP POOL MEMBER"); + return false; +} - EventEmitter{*admin}.emit([&](CasEvent & e) +/// The mark records that this run's claim, tombstone and farewell all held; it does not wait for GC. +bool verifyAndMark(CasOperation & op, const Layout & layout, const String & srid, UInt128 victim_uuid, + uint64_t admin_epoch, const Tombstone & tombstone, std::vector & warnings) +{ + const size_t warnings_before = warnings.size(); + const String owner_key = layout.ownerKey(srid); + try { - e.type = CasEventType::MemberDecommission; - e.outcome = "begin"; - e.reason = "operator decommission of pool member"; - e.detail = {{"server_root_id", victim_srid}}; - }); + const std::optional slot = op.read(layout.mountKey(srid), Retry::standard()); + const std::optional current = op.read(owner_key, Retry::standard()); + if (slot && current) + { + const MountLease body = decodeMountLease(slot->bytes); + const OwnerObject owner = decodeOwner(current->bytes); + const bool farewell_held = body.isHeldBy(victim_uuid, admin_epoch) && !body.gc_fenced && body.isCleanRelease(); + const bool tombstone_held = owner.server_uuid == victim_uuid && owner.retired_at_ms + && owner.owner_life == tombstone.owner.owner_life; + if (farewell_held && tombstone_held) + { + const OwnerObject marked_owner = nextOwnerLife(owner, owner.retired_at_ms, true); + const WriteResult marked = op.replace(owner_key, encodeOwner(marked_owner), current->etag, Retry::standard()); + if (std::holds_alternative(marked)) + return true; + warnings.push_back("the verified mark was not written (" + whyNotWritten(marked) + ")"); + } + else + warnings.push_back(farewell_held ? "the owner is not this run's tombstone" : "the slot does not hold this run's farewell"); + } + } + catch (...) + { + warnings.push_back("verification failed: " + getCurrentExceptionMessage(/*with_stacktrace=*/false)); + } + if (!ownerHoldsVerifiedRetirement(op, layout, srid, tombstone, warnings)) + return false; + /// The owner re-read found the mark: the failure above, e.g. an unclear mark write, needs no rerun. + for (size_t i = warnings_before; i < warnings.size(); ++i) + LOG_INFO(getLogger("CasDecommission"), "CAS decommission '{}': settled by the owner re-read: {}", srid, warnings[i]); + warnings.resize(warnings_before); + return true; +} + +/// The drain on the admin's fenced mount plane: victim namespaces, manifest debris, staging, roots. Per-object +/// failures land in `report.warnings`; a namespace that changed or lost its checkpoint throws. +void drainVictim(Pool & admin, const CasRefCatalog::Snapshot & catalog_cut, const String & victim_srid, + DecommissionReport & report, bool & gc_round_needed) +{ + CasOperation op = admin.mountRequests().admit(); /// The pre-impersonation catalog cut is the complete ownership universe. Physical life keys carry /// no logical path, and raw string prefixes such as `victim` must not select the distinct owner @@ -217,7 +367,7 @@ DecommissionReport decommissionPoolMember(BackendPtr backend, PoolConfig config, /// Refuse a same-name lifecycle move that landed after the immutable selection cut. The /// exact-life overloads below also pin recovery to `life`, closing the race after this check: /// a later replacement can never redirect a removal to its new incarnation. - const CasRefCatalog::Snapshot current_catalog = CasRefCatalog::read(op, admin->layout()); + const CasRefCatalog::Snapshot current_catalog = CasRefCatalog::read(op, admin.layout()); const auto current_entry = std::find_if( current_catalog.catalog.entries.begin(), current_catalog.catalog.entries.end(), [&](const CatalogEntry & entry) { return entry.ns.string() == ns_str; }); @@ -229,7 +379,7 @@ DecommissionReport decommissionPoolMember(BackendPtr backend, PoolConfig config, if (selected_entry.state == NsState::Removing) { - if (!op.head(admin->layout().refCkptKey(life), Retry::standard()).has_value()) + if (!op.head(admin.layout().refCkptKey(life), Retry::standard()).has_value()) throw Exception(ErrorCodes::CORRUPTED_DATA, "ca-decommission: namespace '{}' is Removing but its exact checkpoint is absent; " "the catalog row remains owned and the victim slot cannot be retired", @@ -238,13 +388,13 @@ DecommissionReport decommissionPoolMember(BackendPtr backend, PoolConfig config, /// `dropNamespace` is the sole terminal writer. On an already-complete removal this is an /// idempotent observation; on a pre-terminal `Removing` life it resumes the exact terminal /// append under the administrative writer fence. Catalog deletion remains GC's job. - (void)admin->dropNamespace(life); + (void)admin.dropNamespace(life); ++report.namespaces_already_removed; gc_round_needed = true; continue; } - const auto stats = admin->dropNamespace(life); + const auto stats = admin.dropNamespace(life); ++report.namespaces_removed; report.committed_refs_removed += stats.committed_refs; report.precommits_removed += stats.precommits; @@ -252,7 +402,7 @@ DecommissionReport decommissionPoolMember(BackendPtr backend, PoolConfig config, if (selected_entry.state != NsState::Creating) gc_round_needed = true; - EventEmitter{*admin}.emit([&](CasEvent & e) + EventEmitter{admin}.emit([&](CasEvent & e) { e.type = CasEventType::MemberDecommission; e.outcome = "namespace_removed"; @@ -270,248 +420,167 @@ DecommissionReport decommissionPoolMember(BackendPtr backend, PoolConfig config, /// Group the listed keys by namespace and build prefix so each group can use the exact-token orphan /// sweep while the mount body still supplies its authority. { - const String debris_prefix = admin->layout().casManifestsServerPrefix(victim_srid); + const String debris_prefix = admin.layout().casManifestsServerPrefix(victim_srid); std::set> groups; /// (namespace, writer epoch, build sequence) op.forEachListedKey(debris_prefix, [&](const ListedKey & listed) { - if (const auto parsed = admin->layout().parseManifestKey(listed.key)) + if (const auto parsed = admin.layout().parseManifestKey(listed.key)) groups.emplace(parsed->root_namespace.string(), parsed->ref.writer_epoch, parsed->ref.build_sequence); return true; }, Retry::standard()); for (const auto & [ns_str, writer_epoch, build_sequence] : groups) report.manifest_debris_removed += sweepNamespace( - *admin, RootNamespace(ns_str), BuildPrefix{writer_epoch, build_sequence}, &report.warnings); + admin, RootNamespace(ns_str), BuildPrefix{writer_epoch, build_sequence}, &report.warnings); } /// Drain the victim's own `/staging//` area. The live-mount staging helper uses /// an `IObjectStorage`, while this command intentionally works at the `Backend` layer, so the same /// prefix is listed and deleted directly. The claim fences the victim's writers during this sweep. - report.staging_objects_removed += deleteListedPrefix( - op, admin->poolConfig().pool_prefix + "/staging/" + victim_srid + "/", report.warnings); + report.staging_objects_removed += deleteListedPrefix(op, admin.layout().stagingPrefix(victim_srid), report.warnings); /// Drain the victim's mountpoint objects. These are loose, non-content-addressed files under /// `Layout::serverRootDataPrefix`; they have no writer epoch of their own, so the claim is what /// prevents a returning victim from racing this deletion. report.mountpoint_objects_removed += deleteListedPrefix( - op, admin->layout().serverRootDataPrefix(victim_srid), report.warnings); - - /// The catalog, not physical debris, owns the slot-retirement decision. A terminal append only - /// moves a row to `Removing`; GC must fold/prune/delete it before the member's ownership anchor can - /// disappear. Capture one exact whole-catalog cut after every drain, then revalidate its incarnation - /// and canonical value immediately before entering the retirement tail. The administrative claim - /// fences the victim writer between those observations. - std::optional retirement_catalog_cut; - if (report.warnings.empty()) + op, admin.layout().serverRootDataPrefix(victim_srid), report.warnings); + + if (!op.admitted()) { - retirement_catalog_cut = CasRefCatalog::read(op, admin->layout()); - const uint64_t victim_owned_count = std::count_if( - retirement_catalog_cut->catalog.entries.begin(), retirement_catalog_cut->catalog.entries.end(), - [&](const CatalogEntry & entry) - { - return entry.ns.string() == victim_srid - || entry.ns.string().starts_with(victim_namespace_prefix); - }); - if (victim_owned_count > 0) - report.warnings.push_back( - "pool member decommission underway: " + std::to_string(victim_owned_count) - + " namespace(s) are still owned by this member; upcoming GC rounds perform the final " - "cleanup — re-run this command afterwards to retire the slot"); + report.warnings.push_back("the admin lost its mount lease during the drain; rerun"); + return; } - /// Retire the slot strictly last and only after a clean drain. Copy the layout before - /// `admin.reset()`: graceful close destroys the `Pool`, while `op` (holding its own shared_ptr to - /// the backend) remains usable to retire the slot objects afterwards. - const Layout layout = admin->layout(); - if (report.warnings.empty()) + /// Rows still owned by the victim after the drain. `Removing` rows are GC's: they count as pending, not + /// as a failure. Anything else appeared after the ownership cut and was not drained: rerun. + const CasRefCatalog::Snapshot after_drain = CasRefCatalog::read(op, admin.layout()); + for (const CatalogEntry & entry : after_drain.catalog.entries) { - const CasRefCatalog::Snapshot fresh_retirement_catalog = CasRefCatalog::read(op, admin->layout()); - if (!retirement_catalog_cut - || fresh_retirement_catalog.etag != retirement_catalog_cut->etag - || fresh_retirement_catalog.catalog != retirement_catalog_cut->catalog) - { - report.warnings.push_back( - "catalog changed after the victim ownership check; refusing slot retirement against a stale cut"); - } + if (entry.ns.string() != victim_srid && !entry.ns.string().starts_with(victim_namespace_prefix)) + continue; + if (entry.state == NsState::Removing) + ++report.namespaces_pending_gc; + else + report.warnings.push_back("namespace '" + entry.ns.string() + "' appeared after the ownership cut and was not drained; rerun"); } - if (report.warnings.empty()) +} + +} + +bool adminClaimStillHeld(CasOperation & op, const Layout & layout, const String & srid, UInt128 victim_uuid, uint64_t admin_epoch) +{ + const std::optional slot = op.read(layout.mountKey(srid), Retry::standard()); + if (!slot) + return false; + const MountLease body = decodeMountLease(slot->bytes); + return body.isHeldBy(victim_uuid, admin_epoch) && !body.gc_fenced && !body.isCleanRelease(); +} + +DecommissionReport decommissionPoolMember(BackendPtr backend, PoolConfig config, + const String & victim_srid, const CasEventSink & sink, + const std::function & request_gc_round, + const std::function & drain_now_fn, + const std::function & drain_sleep_fn) +{ + DecommissionReport report; + report.srid = victim_srid; + bool gc_round_needed = false; + /// A namespace may have reached `Removing` before a later namespace fails closed: wake GC on every exit. + SCOPE_EXIT({ + if (gc_round_needed && request_gc_round) + request_gc_round(); + }); + + /// Validate one immutable ownership cut before impersonating the victim, on the raw backend: no `Pool` + /// exists yet. Namespace selection reuses this exact pre-mutation decision rather than a later read. + CasRequests preflight_requests(backend, Fence::open()); + CasOperation preflight_op = preflight_requests.admit(); + const Layout catalog_layout(config.pool_prefix); + const CasRefCatalog::Snapshot catalog_cut = CasRefCatalog::read(preflight_op, catalog_layout); + catalog_cut.life_index.throwIfAmbiguous("CAS decommission"); + + /// One fake clock and sleep for every plane: the farewell deadline is bound to the boot clock, and a + /// plane on a frozen clock that sleeps for real would never see its retry deadline elapse. + if (drain_now_fn && drain_sleep_fn) { - const String mount_key = layout.mountKey(victim_srid); - const String epoch_key = layout.epochKey(victim_srid); - const String owner_key = layout.ownerKey(victim_srid); - - /// Capture both the epoch value and its exact incarnation while the decommission claim still - /// fences the victim. A successor can only bump this object after the farewell below releases - /// the claim, so this incarnation is the epoch-side successor fence for the retirement tail. - std::optional claimed_epoch; - try - { - claimed_epoch = op.read(epoch_key, Retry::standard()); - if (!claimed_epoch) - report.warnings.push_back("slot capture failed: " + epoch_key + " is absent under the admin claim"); - } - catch (...) - { - report.warnings.push_back("slot capture failed: " + epoch_key + ": " - + getCurrentExceptionMessage(/*with_stacktrace=*/false)); - } + if (!config.boot_ms_fn) + config.boot_ms_fn = drain_now_fn; + if (!config.retry_sleep_fn) + config.retry_sleep_fn = drain_sleep_fn; + } + config.event_sink = sink; + + /// The admin claim. `NoWait` refuses a live victim and accepts a tombstoned owner. + PoolPtr admin = Pool::openForDecommission(std::move(backend), std::move(config), victim_srid); + if (drain_now_fn && drain_sleep_fn) + { + admin->setCasRequestNowFnForTest(drain_now_fn); + admin->setCasRetrySleepForTest(drain_sleep_fn); + } + const UInt128 victim_uuid = admin->poolConfig().server_id; + const uint64_t admin_epoch = admin->writerEpoch(); + const Layout layout = admin->layout(); - /// Graceful close stamps an already-expired lease and the watermark farewell - /// (`min_active_build_sequence = UINT64_MAX`), making the slot `terminated` before its mutable control objects - /// are removed and its owner anchor is tombstoned. - admin.reset(); + /// The control plane: always admitted, so a fenced admin can still confirm, roll back and verify. It keeps + /// its own shared_ptr to the backend and outlives `admin.reset()`. + CasRequests control_requests(admin->poolBackendPtr(), Fence::open()); + if (drain_now_fn && drain_sleep_fn) + { + control_requests.setNowFnForTest(drain_now_fn); + control_requests.setSleepFnForTest(drain_sleep_fn); + } + CasOperation control = control_requests.admit(); - /// Read the farewell immediately after `finishTeardown` wrote it. Its exact incarnation is the - /// mount-side fence: deleting by this incarnation can remove only THIS decommission's farewell, - /// not a successor reclaim. Validate the body against the epoch value captured under the claim - /// so a successor that completed before this read is also recognized and left untouched. - std::optional farewell_mount; - try - { - farewell_mount = op.read(mount_key, Retry::standard()); - if (!farewell_mount) - report.warnings.push_back("slot capture failed: " + mount_key + " farewell is absent"); - } - catch (...) - { - report.warnings.push_back("slot capture failed: " + mount_key + ": " - + getCurrentExceptionMessage(/*with_stacktrace=*/false)); - } + EventEmitter{*admin}.emit([&](CasEvent & e) + { + e.type = CasEventType::MemberDecommission; + e.outcome = "begin"; + e.reason = "operator decommission of pool member"; + e.detail = {{"server_root_id", victim_srid}, {"server_uuid", u128ToHex(victim_uuid)}}; + }); - bool captures_match = claimed_epoch && farewell_mount; - if (captures_match) + /// From here on every exit decides the tombstone's fate (`confirmClaimOrRollBack`). + const Tombstone tombstone = writeTombstone(control, *admin, layout, victim_srid, victim_uuid, admin_epoch, report.warnings); + + bool drain_started = false; + try + { + /// The claim must have held through the tombstone write. + if (!adminClaimStillHeld(control, layout, victim_srid, victim_uuid, admin_epoch)) { - try - { - const ServerEpoch epoch_value = decodeServerEpoch(claimed_epoch->bytes); - const MountLease mount_value = decodeMountLease(farewell_mount->bytes); - captures_match = epoch_value.next_writer_epoch != 0 - && mount_value.writer_epoch == epoch_value.next_writer_epoch - 1 - && mount_value.min_active_build_sequence == std::numeric_limits::max() - && !mount_value.gc_fenced; - if (!captures_match) - { - report.warnings.push_back( - "slot capture failed: " + mount_key - + " is not this decommission's farewell for the epoch captured under the admin claim"); - } - } - catch (...) - { - report.warnings.push_back("slot capture failed while validating " + mount_key + " and " + epoch_key + ": " - + getCurrentExceptionMessage(/*with_stacktrace=*/false)); - captures_match = false; - } + rollBackTombstone(control, layout, victim_srid, tombstone, report.warnings); } - - /// Mount first: if a successor reclaimed it after the farewell capture, the stale farewell - /// incarnation yields `Mismatch` and the tail stops before touching epoch or owner. Epoch - /// second: its under-claim incarnation similarly detects a successor allocation. Before - /// touching owner, re-read both mutable objects: a same-UUID successor can recreate them after - /// both deletes without rewriting the owner identity anchor. Mere presence proves that the slot - /// is live again. Every delete must be explicitly confirmed as `Removed`, and the final owner - /// tombstone rewrite must succeed against the exact incarnation read immediately before it. - /// - /// ACCEPTED RESIDUAL WINDOW (final review, not closed by this recheck): a same-UUID successor - /// can still recreate epoch/mount in the narrow gap strictly AFTER this liveness recheck but - /// BEFORE the owner CAS below reads its own incarnation -- the successor's owner anchor (same - /// server_uuid, not yet retired) then gets tombstoned by this decommission run. The successor's - /// live process is not deleted (only its owner anchor is marked retired), but a LATER restart of - /// that same identity would refuse to reclaim it (claimOwnerOrThrow's tombstone guard). This is - /// a narrow, low-probability window, deliberately not closed here: T5's owner-tombstone design - /// (finding #9) intentionally stopped short of making concurrent decommission-vs-recreate - /// airtight to the microsecond, since that was explicitly not the priority for this fix. - report.slot_removed = false; - if (captures_match && deleteSlotObject(op, mount_key, farewell_mount->etag, report.warnings) - && deleteSlotObject(op, epoch_key, claimed_epoch->etag, report.warnings)) + else { - std::optional current_mount; - std::optional current_epoch; - bool liveness_recheck_succeeded = true; - try - { - current_mount = op.read(mount_key, Retry::standard()); - } - catch (...) - { - report.warnings.push_back("slot liveness recheck failed: " + mount_key + ": " - + getCurrentExceptionMessage(/*with_stacktrace=*/false)); - liveness_recheck_succeeded = false; - } - try - { - current_epoch = op.read(epoch_key, Retry::standard()); - } - catch (...) - { - report.warnings.push_back("slot liveness recheck failed: " + epoch_key + ": " - + getCurrentExceptionMessage(/*with_stacktrace=*/false)); - liveness_recheck_succeeded = false; - } - - if (liveness_recheck_succeeded && (current_mount || current_epoch)) - { - report.warnings.push_back( - "slot delete aborted: successor reappeared after mutable control-object deletion; owner kept"); - } - else if (liveness_recheck_succeeded) - { - try - { - if (const auto owner = op.read(owner_key, Retry::standard())) - { - OwnerObject tombstoned = decodeOwner(owner->bytes); - tombstoned.retired_at_ms = Cas::nowMs(); - /// `op.replace` resolves an ambiguous attempt with a resolve read on its own: - /// a transient transport error here (or one whose response was simply lost) - /// must not be reported as a hard failure when the write actually landed. - /// Unchanged incarnation means the write never applied (legitimately retryable - /// within budget); matching bytes means this exact tombstone already landed - /// (`Committed`, not a failure); a genuine successor reclaim is `Conflict`; - /// `Refused` is the store's own definite answer (a denial, a malformed - /// request, an expired credential) and carries its own code and message, - /// which is worth more here than the generic retry advice below. - WriteResult result = op.replace(owner_key, encodeOwner(tombstoned), owner->etag, Retry::standard()); - if (std::holds_alternative(result)) - report.slot_removed = true; - else if (std::holds_alternative(result)) - report.warnings.push_back( - "slot tombstone failed: " + owner_key - + ": successor reclaimed the owner anchor before this decommission's tombstone write"); - else if (const Refused * refused = std::get_if(&result)) - report.warnings.push_back( - "slot tombstone failed: " + owner_key + ": the store refused the write (" - + std::to_string(refused->store_error) + "): " + refused->message); - else - report.warnings.push_back( - "slot tombstone failed: " + owner_key - + ": tombstone write outcome could not be resolved (retry budget exhausted " - "or the resolve read itself failed) -- rerun the command to retry"); - } - else - report.warnings.push_back( - "slot tombstone failed: " + owner_key + ": object absent before tombstone write"); - } - catch (...) - { - report.warnings.push_back("slot tombstone failed: " + owner_key + ": " - + getCurrentExceptionMessage(/*with_stacktrace=*/false)); - } - } + drain_started = true; + drainVictim(*admin, catalog_cut, victim_srid, report, gc_round_needed); + if (!report.warnings.empty()) + confirmClaimOrRollBack(control, layout, victim_srid, victim_uuid, admin_epoch, tombstone, drain_started, + report.warnings); } } - else + catch (...) + { + /// The farewell `admin`'s destructor writes on unwind is a clean body, which the confirm would read as + /// a lost claim, so the decision is made here, while the admin is alive. + confirmClaimOrRollBack(control, layout, victim_srid, victim_uuid, admin_epoch, tombstone, drain_started, + report.warnings); + logWarnings(victim_srid, report.warnings); + throw; + } + + /// The farewell: it lands while the claim holds and loses to the fence after a lost claim. + /// `MountLeaseRenewer::terminate` reports no outcome, so `verifyAndMark` reads it back. + admin.reset(); + if (report.warnings.empty()) { - report.slot_removed = false; - LOG_WARNING(getLogger("CasDecommission"), - "CAS decommission '{}': drain incomplete ({} warnings) — mount slot kept (terminated); " - "re-run the command to finish", victim_srid, report.warnings.size()); - admin.reset(); /// Graceful close still stamps the farewell, leaving the slot `terminated`. + report.member_retired = verifyAndMark(control, layout, victim_srid, victim_uuid, admin_epoch, tombstone, report.warnings); + if (report.member_retired) + LOG_INFO(getLogger("CasDecommission"), "CAS decommission '{}': retired server_uuid={} at owner life {}", + victim_srid, u128ToHex(victim_uuid), tombstone.owner.owner_life); } + logWarnings(victim_srid, report.warnings); - /// The `end` event is emitted via `sink` directly, not `EventEmitter{*admin}`: `admin` is gone by - /// now. This also means its `warnings` count reflects the FINAL total, including a slot-retirement - /// failure appended just above -- `EventEmitter`'s own zero-cost-when-absent guard is reproduced by - /// the `if (sink)` below. + /// Emitted through `sink` directly: `admin` is gone by now. if (sink) { CasEvent e; @@ -519,9 +588,12 @@ DecommissionReport decommissionPoolMember(BackendPtr backend, PoolConfig config, e.outcome = "end"; e.reason = "decommission finished"; e.detail = {{"server_root_id", victim_srid}, + {"server_uuid", u128ToHex(victim_uuid)}, + {"owner_life", std::to_string(tombstone.owner.owner_life)}, {"namespaces_removed", std::to_string(report.namespaces_removed)}, + {"namespaces_pending_gc", std::to_string(report.namespaces_pending_gc)}, {"warnings", std::to_string(report.warnings.size())}, - {"slot_removed", report.slot_removed ? "1" : "0"}}; + {"member_retired", report.member_retired ? "1" : "0"}}; sink(std::move(e)); } return report; diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.h b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.h index 6832fd9cfe731..484b642868b37 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.h +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/Tools/CasDecommission.h @@ -16,51 +16,58 @@ namespace DB::Cas /// the manifest, staging, and mountpoint drains. Blob bytes are intentionally not reported: removing /// ref edges makes them eligible for ordinary GC, but this operation does not synchronously reclaim /// shared content. -/// -/// A decommission is resumable. A previous run may already have moved namespaces to `Removing`, and a -/// warning means that the corresponding drain was not confirmed. In either case the report lets the -/// caller distinguish work done by this invocation from work observed from an earlier invocation. struct DecommissionReport { String srid; /// The decommissioned member's `server_root_id`. uint64_t namespaces_removed = 0; /// Namespaces erased by this invocation. uint64_t namespaces_already_removed = 0; /// Namespaces already `Removing` on entry. + /// Victim rows still `Removing` when the run ended: GC deletes them; no rerun is needed for them. + uint64_t namespaces_pending_gc = 0; uint64_t committed_refs_removed = 0; /// Committed ref records removed by namespace drops. uint64_t precommits_removed = 0; /// Precommit records removed by namespace drops. uint64_t edge_deltas_emitted = 0; /// The sum of `committed_refs_removed` and `precommits_removed`. uint64_t manifest_debris_removed = 0; /// Eligible manifest objects deleted from old build prefixes. uint64_t staging_objects_removed = 0; /// Objects deleted from the member's staging prefix. uint64_t mountpoint_objects_removed = 0; /// Objects deleted from the member's roots/mountpoint prefix. - bool slot_removed = false; /// Whether mount and epoch were deleted and the owner was tombstoned. - std::vector warnings; /// Drain or slot-retirement failures; a non-empty list keeps the slot. + /// The owner carries a verified retirement: the member is retired, do not rerun. + bool member_retired = false; + /// Why the member is not retired: a drain failure, a lost claim, an unverified outcome (rerun), or an owner + /// that moved on to a later life (read the owner before rerunning). + std::vector warnings; }; -/// Erases all content owned by a permanently dead pool member. The operation first claims the member's -/// slot as an administrative writer; a live lease is refused, and the claim fences the dead member from -/// writing while cleanup runs. It then drops each table namespace through `Pool::dropNamespace`, drains -/// eligible manifest debris, staging objects, and mountpoint objects, and retires the slot only after all -/// drains are confirmed. Namespace drops are idempotent: a rerun resumes any missing terminal append -/// and leaves exact catalog-row deletion to GC. The member slot remains while any catalog entry still -/// belongs to the victim. +/// Whether the slot still holds the admin's claim: the victim's uuid, `admin_epoch`, neither fenced nor +/// farewelled. Renewals change the ETag and `write_attempt_id`, so neither is compared. +bool adminClaimStillHeld(CasOperation & op, const Layout & layout, const String & srid, UInt128 victim_uuid, uint64_t admin_epoch); + +/// Erases all content owned by a permanently dead pool member and retires its owner. Never deletes +/// `owner`, `epoch` or `mount`. The steps: +/// 1. Claim the member's slot as an administrative writer (`Pool::openForDecommission`). A live lease is +/// refused; an owner retired by an earlier run is accepted. +/// 2. Tombstone the owner on the admin's fenced mount plane: the next owner life, retired, keeping an earlier +/// verified mark. No attempt is sent after the admin's lease deadline. +/// 3. Confirm the claim still holds. A claim lost before the drain rolls the tombstone back unless the owner was +/// already retired; a claim lost during the drain keeps it. The same confirm runs on every failed exit before the +/// farewell; a clean drain is checked in step 6. +/// 4. Drain on the admin's fenced mount plane: each victim namespace through `Pool::dropNamespace`, then +/// eligible manifest debris, staging objects and mountpoint objects. +/// 5. Write the admin's clean farewell. +/// 6. Read the slot and the owner back and mark the retirement verified (the next owner life). Only a +/// verified mark reports `member_retired`. /// -/// This is a writer operation, not GC: it emits the normal ref-edge deltas and does not invent ref -/// transitions. Per-object drain failures are recorded in `DecommissionReport::warnings` and leave the -/// terminated slot as a resume anchor; other failures, including refusal to claim the member, propagate -/// as exceptions. When set, `sink` receives `MemberDecommission` audit events for the run's begin, -/// per-namespace, and end milestones. +/// A rerun resumes over the run's own unverified tombstone; a verified retirement is rerun-safe and +/// reports retired again. Catalog rows GC still has to delete count as `namespaces_pending_gc`, not as a +/// failure. Per-object drain failures are recorded in `DecommissionReport::warnings`; other failures, +/// including refusal to claim the member, propagate as exceptions. When set, `sink` receives +/// `MemberDecommission` audit events for the run's begin, per-namespace, and end milestones. /// -/// `drain_now_fn`/`drain_sleep_fn`, when both set, replace the clock the drain's own request engine -/// paces its retries on -- the engine this function opens is a standalone one over the instrumented -/// backend, not one of `Pool`'s planes, so `Pool::setCasRetrySleepForTest` cannot reach it. A test -/// driving a latched per-object fault to `Retry::standard()`'s own give-up needs this seam, or it pays -/// the real 90-second deadline. +/// This is a writer operation, not GC: it emits the normal ref-edge deltas and does not invent ref +/// transitions. /// -/// `drain_now_fn`/`drain_sleep_fn`, when BOTH set, also become the opened `Pool`'s own boot clock and -/// retry sleep (`PoolConfig::boot_ms_fn`/`retry_sleep_fn`) whenever the caller left those fields unset: -/// the mount lease's farewell deadline is bound to the boot clock, and `Pool::openForDecommission`'s own -/// mount/farewell/GC planes are constructed with it too, so a caller that fakes only the standalone -/// engine's clock must not end up comparing it against the real one, or -- worse -- against a plane that -/// shares the frozen clock but still sleeps for real between retries. +/// `drain_now_fn`/`drain_sleep_fn`, when both set, replace the clock and retry sleep of the control-plane +/// request engine this function opens and of the admin `Pool`'s planes, and become the `Pool`'s boot clock +/// and retry sleep (`PoolConfig::boot_ms_fn`/`retry_sleep_fn`) whenever the caller left those unset: the +/// farewell deadline is bound to the boot clock, so one fake clock must drive all of them. DecommissionReport decommissionPoolMember(BackendPtr backend, PoolConfig config, const String & victim_srid, const CasEventSink & sink = {}, const std::function & request_gc_round = {}, diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp index f2d67cea6728c..9c2eb04456f35 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.cpp @@ -103,6 +103,27 @@ namespace ErrorCodes namespace { +/// `WriteBufferFromS3` and `copyS3File` reissue the identical request on `NO_SUCH_KEY` in their own +/// loops above the client, so neither a client retry profile nor single-attempt creation bounds them. +void applyWriteErrorRetryOverride(S3::S3RequestSettings & request_settings, const WriteSettings & write_settings) +{ + if (write_settings.object_storage_single_attempt_object_creation) + request_settings[S3RequestSetting::max_unexpected_write_error_retries] = 1; + else if (write_settings.s3_max_unexpected_write_error_retries_override) + request_settings[S3RequestSetting::max_unexpected_write_error_retries] + = write_settings.s3_max_unexpected_write_error_retries_override; +} + +void requireNativeOnlyForSingleAttemptCreation(const WriteSettings & write_settings) +{ + if (write_settings.object_storage_single_attempt_object_creation + && write_settings.object_storage_copy_mode != ObjectStorageCopyMode::NativeOnly) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "A copy with single-attempt object creation requires NativeOnly copy mode: the read-write " + "fallback would write through the retrying client"); +} + template void throwIfError(const Aws::Utils::Outcome & response) { @@ -440,15 +461,7 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN request_settings[S3RequestSetting::min_upload_part_size] = cap; } - if (write_settings.s3_max_unexpected_write_error_retries_override) - { - /// WriteBufferFromS3's OWN retry loop (makeSinglepartUpload/completeMultipartUpload) reissues - /// the identical request — WITH its If-None-Match/If-Match condition — on a NO_SUCH_KEY - /// response; this sits ABOVE the S3 client, so a client-level profile override does not bound - /// it. See WriteSettings. - request_settings[S3RequestSetting::max_unexpected_write_error_retries] - = write_settings.s3_max_unexpected_write_error_retries_override; - } + applyWriteErrorRetryOverride(request_settings, write_settings); ThreadPoolCallbackRunnerUnsafe scheduler; if (write_settings.s3_allow_parallel_part_upload) @@ -475,7 +488,8 @@ std::unique_ptr S3ObjectStorage::writeObject( /// NOLIN std::move(blob_storage_log), attributes, std::move(scheduler), - disk_write_settings); + disk_write_settings, + objectCreationClient(write_settings)); } @@ -999,12 +1013,17 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT IObjectStorage & object_storage_to, std::optional object_to_attributes) { + /// A non-S3 destination is rejected here under the flag: `NativeOnly` throws `NOT_IMPLEMENTED` below. + requireNativeOnlyForSingleAttemptCreation(write_settings); + /// Shortcut for S3 if (auto * dest_s3 = dynamic_cast(&object_storage_to); dest_s3 != nullptr) { auto current_client = dest_s3->client.get(); auto settings_ptr = s3_settings.get(); auto size = S3::getObjectSize(*client.get(), uri.bucket, object_from.remote_path, {}); + S3::S3RequestSettings request_settings = settings_ptr->request_settings; + applyWriteErrorRetryOverride(request_settings, write_settings); auto scheduler = threadPoolCallbackRunnerUnsafe(getThreadPoolWriter(), ThreadName::S3_COPY_POOL); const auto read_settings_to_use = patchSettings(read_settings); @@ -1018,13 +1037,14 @@ void S3ObjectStorage::copyObjectToAnotherObjectStorage( // NOLINT /*dest_s3_client=*/current_client, /*dest_bucket=*/dest_s3->uri.bucket, /*dest_key=*/object_to.remote_path, - settings_ptr->request_settings, + request_settings, read_settings_to_use, BlobStorageLogWriter::create(disk_name), scheduler, [&, this]{ return readObject(object_from, read_settings_to_use);}, object_to_attributes, - write_settings.object_storage_copy_mode); + write_settings.object_storage_copy_mode, + dest_s3->objectCreationClient(write_settings)); return; } catch (S3Exception & exc) @@ -1077,9 +1097,14 @@ void S3ObjectStorage::copyObject( // NOLINT "(allow_native_copy=false) for object storage {}", getName()); + requireNativeOnlyForSingleAttemptCreation(write_settings); + auto current_client = client.get(); auto settings_ptr = s3_settings.get(); auto size = S3::getObjectSize(*current_client, uri.bucket, object_from.remote_path, {}); + + S3::S3RequestSettings request_settings = settings_ptr->request_settings; + applyWriteErrorRetryOverride(request_settings, write_settings); auto scheduler = threadPoolCallbackRunnerUnsafe(getThreadPoolWriter(), ThreadName::S3_COPY_POOL); const auto read_settings_to_use = patchSettings(read_settings); @@ -1091,13 +1116,14 @@ void S3ObjectStorage::copyObject( // NOLINT /*dest_s3_client=*/current_client, /*dest_bucket=*/uri.bucket, /*dest_key=*/object_to.remote_path, - settings_ptr->request_settings, + request_settings, read_settings_to_use, BlobStorageLogWriter::create(disk_name), scheduler, [&, this]{ return readObject(object_from, read_settings_to_use);}, object_to_attributes, - write_settings.object_storage_copy_mode); + write_settings.object_storage_copy_mode, + objectCreationClient(write_settings)); } void S3ObjectStorage::shutdown() @@ -1255,7 +1281,8 @@ std::shared_ptr S3ObjectStorage::tryGetS3StorageClient() return client.get(); } -std::shared_ptr S3ObjectStorage::getSingleAttemptClient(uint64_t request_timeout_ms, uint64_t connect_timeout_cap_ms) const +std::shared_ptr S3ObjectStorage::getSingleAttemptClient( + uint64_t request_timeout_ms, uint64_t connect_timeout_cap_ms, bool disable_adaptive_timeouts) const { auto base = client.get(); std::lock_guard lock(single_attempt_client_mutex); @@ -1265,7 +1292,7 @@ std::shared_ptr S3ObjectStorage::getSingleAttemptClient(uint64 single_attempt_client_base = base; } - const auto cache_key = std::make_pair(request_timeout_ms, connect_timeout_cap_ms); + const auto cache_key = std::make_tuple(request_timeout_ms, connect_timeout_cap_ms, disable_adaptive_timeouts); if (auto it = single_attempt_clients.find(cache_key); it != single_attempt_clients.end()) return it->second; @@ -1282,6 +1309,8 @@ std::shared_ptr S3ObjectStorage::getSingleAttemptClient(uint64 if (request_timeout_ms != 0) cfg.requestTimeoutMs = static_cast(request_timeout_ms); + if (disable_adaptive_timeouts) + cfg.s3_use_adaptive_timeouts = false; /// One TCP/TLS connect may not cost more than the cap the mount froze at open: the engine reserves /// attempt + 2 × cap per envelope, and a reloaded base client with a wider connect timeout must not @@ -1304,18 +1333,30 @@ std::shared_ptr S3ObjectStorage::getSingleAttemptClient(uint64 bool S3ObjectStorage::hasSingleAttemptClientForTest(uint64_t request_timeout_ms, uint64_t connect_timeout_cap_ms) const { std::lock_guard lock(single_attempt_client_mutex); - return single_attempt_clients.contains(std::make_pair(request_timeout_ms, connect_timeout_cap_ms)); + return single_attempt_clients.contains(std::make_tuple(request_timeout_ms, connect_timeout_cap_ms, false)); } std::shared_ptr S3ObjectStorage::clientForRetryProfile(const ObjectStorageControlRequest & request) const { /// getSingleAttemptClient is only invoked when actually selected, so an ordinary request never /// pays for building or locking the clone. + /// The clone keeps the disk's adaptive timeouts: the caller reissues these requests itself and numbers + /// its reissues, and the HTTP client gives attempt 2 onwards the configured timeouts. if (request.profile == ObjectStorageRetryProfile::SingleAttempt) return getSingleAttemptClient(request.attempt_timeout_ms, request.connect_timeout_cap_ms); return client.get(); } +std::shared_ptr S3ObjectStorage::objectCreationClient(const WriteSettings & write_settings) const +{ + if (!write_settings.object_storage_single_attempt_object_creation) + return nullptr; + /// The creating request has no second attempt, so it must not run under the first-attempt timeouts. + return getSingleAttemptClient( + write_settings.object_storage_attempt_timeout_ms, write_settings.object_storage_connect_timeout_cap_ms, + /*disable_adaptive_timeouts=*/true); +} + bool S3ObjectStorage::tryRefreshCredentialsViaCallback() { fiu_do_on(FailPoints::object_storage_force_refresh_callback_success, { return true; }); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index f0671586551ca..50229693f275f 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -214,13 +215,15 @@ class S3ObjectStorage : public IObjectStorage /// disk client rotates (applyNewSettings/credentials refresh) — the cached clone is keyed by the /// base client's identity, so a stale clone can never outlive a rotation. /// `request_timeout_ms` overrides the clone's send/receive inactivity bound; 0 keeps the disk's. - /// `connect_timeout_cap_ms` caps the clone's connect timeout (0 = no cap); the cache key is the - /// pair, so two callers asking for the same request timeout but different caps get distinct clones. - std::shared_ptr getSingleAttemptClient(uint64_t request_timeout_ms, uint64_t connect_timeout_cap_ms = 0) const; - - /// True iff a clone for exactly this (request timeout, connect cap) pair is already cached -- - /// never builds one. Lets a test prove dispatch used a SPECIFIC key (and no other) without ever - /// creating a clone itself and without measuring anything. + /// `connect_timeout_cap_ms` caps the clone's connect timeout (0 = no cap). `disable_adaptive_timeouts` + /// turns off the adaptive first-attempt timeouts, which assume a second attempt with the configured + /// ones. The cache key is all three, so callers asking for different bounds get distinct clones. + std::shared_ptr getSingleAttemptClient( + uint64_t request_timeout_ms, uint64_t connect_timeout_cap_ms = 0, bool disable_adaptive_timeouts = false) const; + + /// True iff a clone for exactly this (request timeout, connect cap) pair, with the disk's adaptive + /// timeouts setting, is already cached -- never builds one. Lets a test prove dispatch used a SPECIFIC + /// key (and no other) without ever creating a clone itself and without measuring anything. bool hasSingleAttemptClientForTest(uint64_t request_timeout_ms, uint64_t connect_timeout_cap_ms) const; private: @@ -245,6 +248,9 @@ class S3ObjectStorage : public IObjectStorage std::shared_ptr clientForRetryProfile(const ObjectStorageControlRequest & request) const; + /// The single-attempt client for the object-creating requests of a write that asks for it, else null. + std::shared_ptr objectCreationClient(const WriteSettings & write_settings) const; + /// Runs `fn` and, if it failed because the vended credentials expired, refreshes this disk's /// client and runs it once more. `fn` must re-read the client itself, so the second run signs /// with the refreshed one. @@ -288,10 +294,10 @@ class S3ObjectStorage : public IObjectStorage std::atomic pinned_generation_dialect{-1}; /// -1 unpinned, 0 pinned ETag, 1 pinned generation mutable std::mutex single_attempt_client_mutex; - /// One clone per (requested timeout, connect cap) pair: the verbs of one operation ask for - /// different bounds, and a single slot would rebuild a whole S3 client (and lose its connection - /// pool) on every alternation between them. - mutable std::map, std::shared_ptr> single_attempt_clients; + /// One clone per (requested timeout, connect cap, adaptive timeouts disabled) key: the verbs of one + /// operation ask for different bounds, and a single slot would rebuild a whole S3 client (and lose + /// its connection pool) on every alternation between them. + mutable std::map, std::shared_ptr> single_attempt_clients; /// The base client the cached clones above were built from. Deliberately held as a shared_ptr (not /// a raw pointer): a raw pointer would be compared for identity AFTER the object it once pointed /// to could have been freed and a new client reallocated at the same address by an unrelated diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index 441451ad5ab37..130286306ac0e 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -143,7 +143,7 @@ catch (...) } DiskSelectorPtr DiskSelector::updateFromConfig( - const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context) const + const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context, const NameSet & disks_to_skip) const { assertInitialized(); @@ -166,6 +166,14 @@ DiskSelectorPtr DiskSelector::updateFromConfig( auto disk_config_prefix = config_prefix + "." + disk_name; if (!result->getDisksMap().contains(disk_name)) { + if (disks_to_skip.contains(disk_name)) + { + LOG_WARNING( + getLogger("DiskSelector"), + "Disk {} is not created from the configuration: a custom disk with this name is being created", + backQuote(disk_name)); + continue; + } auto created_disk = factory.create( disk_name, config, disk_config_prefix, context, result->getDisksMap(), /*attach*/ false, /*custom_disk*/ false, skip_types); if (created_disk) diff --git a/src/Disks/DiskSelector.h b/src/Disks/DiskSelector.h index acf491daa1c3e..f85820db1bfe9 100644 --- a/src/Disks/DiskSelector.h +++ b/src/Disks/DiskSelector.h @@ -29,10 +29,13 @@ class DiskSelector using DiskValidator = std::function; void initialize(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context, DiskValidator disk_validator = {}); + /// Configured disks named in `disks_to_skip` are not created by this reload: their names are being + /// created as custom disks, and creating them here could open the same storage twice. DiskSelectorPtr updateFromConfig( const Poco::Util::AbstractConfiguration & config, const String & config_prefix, - ContextPtr context) const; + ContextPtr context, + const NameSet & disks_to_skip = {}) const; /// Get disk by name DiskPtr get(const String & name) const; diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index 28565c7df6d4f..4e7bae901c11b 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -247,6 +247,10 @@ void IDisk::checkAccess() throw Exception(ErrorCodes::LOGICAL_ERROR, "Server UUID is not initialized"); const String path = fmt::format("clickhouse_access_check_{}", toString(server_uuid)); + /// Disks created in parallel (`Context::getOrCreateDisk`) can share a backing store, and every + /// probe writes, reads and removes this one name: overlapping probes see each other's file. + static std::mutex probe_mutex; + std::lock_guard lock(probe_mutex); checkAccessImpl(path); } diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index eab83e5abd0f3..0fd6c12415ddc 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -483,10 +483,44 @@ bool StoragePolicy::containsVolume(const String & volume_name) const return volume_index_by_volume_name.contains(volume_name); } +namespace +{ + +/// Returns the first disk of `disk_names` that the policy config names, or an empty string. +String findNamedDisk(const Poco::Util::AbstractConfiguration & config, const String & policy_prefix, const NameSet & disk_names) +{ + if (disk_names.empty()) + return {}; + + const String volumes_prefix = policy_prefix + ".volumes"; + if (!config.has(volumes_prefix)) + return {}; + + Poco::Util::AbstractConfiguration::Keys volumes; + config.keys(volumes_prefix, volumes); + for (const auto & volume : volumes) + { + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(volumes_prefix + "." + volume, keys); + for (const auto & key : keys) + { + if (!startsWith(key, "disk")) + continue; + auto disk_name = config.getString(volumes_prefix + "." + volume + "." + key); + if (disk_names.contains(disk_name)) + return disk_name; + } + } + return {}; +} + +} + StoragePolicySelector::StoragePolicySelector( const Poco::Util::AbstractConfiguration & config, const String & config_prefix, - DiskSelectorPtr disks) + DiskSelectorPtr disks, + const NameSet & disks_to_skip) { Poco::Util::AbstractConfiguration::Keys keys; config.keys(config_prefix, keys); @@ -504,12 +538,22 @@ StoragePolicySelector::StoragePolicySelector( * else */ + if (auto skipped_disk = findNamedDisk(config, config_prefix + "." + name, disks_to_skip); !skipped_disk.empty()) + { + LOG_WARNING( + getLogger("StoragePolicySelector"), + "Storage policy {} is not loaded: disk {} is being created as a custom disk", + backQuote(name), + backQuote(skipped_disk)); + continue; + } + policies.emplace(name, std::make_shared(name, config, config_prefix + "." + name, disks)); LOG_INFO(getLogger("StoragePolicySelector"), "Storage policy {} loaded", backQuote(name)); } /// Add default policy if it isn't explicitly specified. - if (!policies.contains(DEFAULT_STORAGE_POLICY_NAME)) + if (!policies.contains(DEFAULT_STORAGE_POLICY_NAME) && findNamedDisk(config, config_prefix + "." + DEFAULT_STORAGE_POLICY_NAME, disks_to_skip).empty()) { auto default_policy = std::make_shared(DEFAULT_STORAGE_POLICY_NAME, config, config_prefix + "." + DEFAULT_STORAGE_POLICY_NAME, disks); policies.emplace(DEFAULT_STORAGE_POLICY_NAME, std::move(default_policy)); @@ -517,9 +561,14 @@ StoragePolicySelector::StoragePolicySelector( } -StoragePolicySelectorPtr StoragePolicySelector::updateFromConfig(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, DiskSelectorPtr disks, Strings & new_disks) const +StoragePolicySelectorPtr StoragePolicySelector::updateFromConfig( + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + DiskSelectorPtr disks, + Strings & new_disks, + const NameSet & disks_to_skip) const { - std::shared_ptr result = std::make_shared(config, config_prefix, disks); + std::shared_ptr result = std::make_shared(config, config_prefix, disks, disks_to_skip); std::set disks_before_reload; std::set disks_after_reload; /// First pass, check. @@ -528,9 +577,14 @@ StoragePolicySelectorPtr StoragePolicySelector::updateFromConfig(const Poco::Uti if (!name.starts_with(TMP_STORAGE_POLICY_PREFIX)) { if (!result->policies.contains(name)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage policy {} is missing in new configuration", backQuote(name)); + { + if (findNamedDisk(config, config_prefix + "." + name, disks_to_skip).empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage policy {} is missing in new configuration", backQuote(name)); - policy->checkCompatibleWith(result->policies[name]); + result->policies[name] = policy; + } + else + policy->checkCompatibleWith(result->policies[name]); } for (const auto & disk : policy->getDisks()) @@ -543,7 +597,7 @@ StoragePolicySelectorPtr StoragePolicySelector::updateFromConfig(const Poco::Uti for (const auto & [name, policy] : policies) { /// Do not reload from config temporary storage policy, because it is not present in config. - if (name.starts_with(TMP_STORAGE_POLICY_PREFIX)) + if (name.starts_with(TMP_STORAGE_POLICY_PREFIX) || result->policies[name] == policy) result->policies[name] = policy; else result->policies[name] = std::make_shared(policy, config, config_prefix + "." + name, disks); diff --git a/src/Disks/StoragePolicy.h b/src/Disks/StoragePolicy.h index eef7dc1aef764..8e79966f9ff1d 100644 --- a/src/Disks/StoragePolicy.h +++ b/src/Disks/StoragePolicy.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -118,9 +119,20 @@ class StoragePolicySelector public: static constexpr auto TMP_STORAGE_POLICY_PREFIX = "__"; - StoragePolicySelector(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, DiskSelectorPtr disks); + /// Policies that name a disk of `disks_to_skip` are left out: that disk is not available yet. + StoragePolicySelector( + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + DiskSelectorPtr disks, + const NameSet & disks_to_skip = {}); - StoragePolicySelectorPtr updateFromConfig(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, DiskSelectorPtr disks, Strings & new_disks) const; + /// An existing policy that names a disk of `disks_to_skip` in the new configuration keeps its previous definition. + StoragePolicySelectorPtr updateFromConfig( + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + DiskSelectorPtr disks, + Strings & new_disks, + const NameSet & disks_to_skip = {}) const; /// Policy by name StoragePolicyPtr get(const String & name) const; diff --git a/src/Disks/tests/S3ObjectCreationTestStorage.h b/src/Disks/tests/S3ObjectCreationTestStorage.h new file mode 100644 index 0000000000000..ec9fd8b1ea6d1 --- /dev/null +++ b/src/Disks/tests/S3ObjectCreationTestStorage.h @@ -0,0 +1,109 @@ +#pragma once + +#include "config.h" + +#if USE_AWS_S3 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace DB::S3RequestSetting +{ +extern const S3RequestSettingsUInt64 max_single_operation_copy_size; +extern const S3RequestSettingsUInt64 max_single_part_upload_size; +extern const S3RequestSettingsUInt64 min_upload_part_size; +} + +/// `S3ObjectStorage` instances for tests against `CountingS3::CountingS3Server`. +namespace S3ObjectCreationTestStorage +{ + +/// A genuine `S3ObjectStorage` whose client retries a retryable failure twice, as a disk's client does. +/// `adaptive_timeouts` gives each request's first attempt the short adaptive timeouts, as on a disk +/// with the default `use_adaptive_timeouts`. +inline std::shared_ptr makeRetryingStorage( + const std::string & endpoint, const std::function & adjust_settings = {}, + bool adaptive_timeouts = false, long request_timeout_ms = 10000) +{ + /// The client configuration keeps a reference to the filter. + static const DB::RemoteHostFilter remote_host_filter; + DB::S3::PocoHTTPClientConfiguration cfg = DB::S3::ClientFactory::instance().createClientConfiguration( + "us-east-1", + remote_host_filter, + /* s3_max_redirects = */ 100, + DB::S3::PocoHTTPClientConfiguration::RetryStrategy{.max_retries = 2, .initial_delay_ms = 1, .max_delay_ms = 1}, + /* s3_slow_all_threads_after_network_error = */ false, + /* s3_slow_all_threads_after_retryable_error = */ false, + /* enable_s3_requests_logging = */ false, + /* for_disk_s3 = */ true, + /* opt_disk_name = */ {}, + /* request_throttler = */ {}); + cfg.endpointOverride = endpoint; + cfg.connectTimeoutMs = 10000; + cfg.requestTimeoutMs = request_timeout_ms; + cfg.s3_use_adaptive_timeouts = adaptive_timeouts; + /// A fresh connection per request: a pooled one can outlive this test's ephemeral-port server. + cfg.http_keep_alive_timeout = 0; + auto client = DB::S3::ClientFactory::instance().create( + cfg, + DB::S3::ClientSettings{ + .use_virtual_addressing = false, + .disable_checksum = false, + .gcs_issue_compose_request = false, + .is_s3express_bucket = false, + }, + "ACCESS_KEY_ID", "SECRET_ACCESS_KEY", "", {}, {}, DB::S3::CredentialsConfiguration{}); + + auto settings = std::make_unique(); + if (adjust_settings) + adjust_settings(*settings); + return std::make_shared( + std::move(client), std::move(settings), DB::S3::URI(endpoint + "/test-bucket/"), DB::S3Capabilities{}, + DB::ObjectStorageKeyGeneratorPtr{}, "disk"); +} + +/// The `DB::Exception` code `fn` throws: 0 when it returns, -1 for any other exception. +inline int thrownCode(const std::function & fn) +{ + try + { + fn(); + } + catch (const DB::Exception & e) + { + return e.code(); + } + catch (...) + { + return -1; + } + return 0; +} + +/// A body above the single-part ceiling that fits one part, so a write is exactly one `UploadPart`. +inline void forceOnePartMultipart(DB::S3Settings & settings) +{ + settings.request_settings[DB::S3RequestSetting::max_single_part_upload_size] = 16; + settings.request_settings[DB::S3RequestSetting::min_upload_part_size] = 1024 * 1024; +} + +/// A staged object above the single-operation copy ceiling that fits one part, so a copy is exactly one +/// `UploadPartCopy`. +inline void forceOnePartMultipartCopy(DB::S3Settings & settings) +{ + settings.request_settings[DB::S3RequestSetting::max_single_operation_copy_size] = 16; + settings.request_settings[DB::S3RequestSetting::min_upload_part_size] = 1024 * 1024; +} + +} + +#endif diff --git a/src/Disks/tests/cas_test_helpers.h b/src/Disks/tests/cas_test_helpers.h index 3a4f870e4867c..82a74f62dc298 100644 --- a/src/Disks/tests/cas_test_helpers.h +++ b/src/Disks/tests/cas_test_helpers.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -27,20 +28,27 @@ #include #include #include +#include +#include #include #include #include #include +#include + #include +#include #include #include #include /// For `ChunkFaultBackend`'s `DefiniteFailure` mode, which needs a real S3-classified error, and for /// the ambiguity it raises otherwise. #include +#include #include +#include #include #include @@ -53,8 +61,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -80,6 +90,37 @@ namespace DB::ErrorCodes namespace DB::Cas::tests { +/// Redirects the named logger into a string for the scope, at the given level. +class ScopedLoggerCapture +{ +public: + ScopedLoggerCapture(const String & logger_name, const String & level) + : logger(getLogger(logger_name)) + , channel(new Poco::StreamChannel(stream)) + , old_channel(logger->getChannel(), /*shared=*/true) + , old_level(logger->getLevel()) + { + logger->setChannel(channel.get()); + logger->setLevel(level); + } + + ~ScopedLoggerCapture() + { + logger->setChannel(old_channel); + logger->setLevel(old_level); + } + + String captured() const { return stream.str(); } + +private: + LoggerPtr logger; + std::ostringstream stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + Poco::AutoPtr channel; + /// A real reference (shared=true), so the parked previous channel cannot die while ours is installed. + Poco::AutoPtr old_channel; + int old_level; +}; + /// A `CasRequests` over an always-open fence, for a fixture that has a backend but no mounted pool. /// Every operation admitted from it holds a reference to it, so it must be named and outlive them. inline DB::Cas::CasRequests openRequestsForTest(DB::Cas::BackendPtr backend) @@ -94,6 +135,49 @@ inline DB::Cas::CasRequests openRequestsForTest(DB::Cas::Backend & backend) return openRequestsForTest(DB::Cas::BackendPtr(std::shared_ptr(), &backend)); } +/// Routes one logger into memory at `level` for the scope, then restores its channel and level. +class ScopedLogCapture +{ +public: + ScopedLogCapture(const String & logger_name, const String & level) + : logger(getLogger(logger_name)) + , channel(new Poco::StreamChannel(stream)) + , old_channel(logger->getChannel(), /*shared=*/true) + , old_level(logger->getLevel()) + { + logger->setChannel(channel.get()); + logger->setLevel(level); + } + + ~ScopedLogCapture() + { + logger->setChannel(old_channel); + logger->setLevel(old_level); + } + + ScopedLogCapture(const ScopedLogCapture &) = delete; + ScopedLogCapture & operator=(const ScopedLogCapture &) = delete; + + String captured() const { return stream.str(); } + + size_t count(std::string_view needle) const + { + const String haystack = captured(); + size_t n = 0; + for (size_t pos = 0; (pos = haystack.find(needle, pos)) != String::npos; pos += needle.size()) + ++n; + return n; + } + +private: + LoggerPtr logger; + std::ostringstream stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + Poco::AutoPtr channel; + /// `shared=true`: the parked channel is not ours, so take a reference instead of stealing one. + Poco::AutoPtr old_channel; + int old_level; +}; + /// An open-fence operation together with the `CasRequests` it refers to, for a fixture that holds a /// backend and needs to call a production entry point taking a `CasOperation &`. Neither copyable nor /// movable: the operation points at the member beside it. @@ -2279,6 +2363,32 @@ class HintHoleBackendOn : public Base /// The plain form, over a bare `InMemoryBackend`. using HintHoleBackend = HintHoleBackendOn; +/// GC's fence-out applied directly to the mount lease: preserve the body, set `gc_fenced`, bump `seq`. +/// A following `tryRemountOnce` reclaims a fresh incarnation. +inline void fenceOutMountForTest(DB::Cas::Backend & backend, const String & mount_key) +{ + OperationForTest op(backend); + const auto got = (*op).read(mount_key, DB::Cas::Retry::once()); + ASSERT_TRUE(got.has_value()); + const DB::Cas::MountLease fenced = DB::Cas::decodeMountLease(got->bytes).fencedCopy(); + const auto put = (*op).replace(mount_key, DB::Cas::encodeMountLease(fenced), got->etag, DB::Cas::Retry::once()); + ASSERT_TRUE(std::holds_alternative(put)); +} + +/// A decommission tombstone by hand: the next owner life, retired, keeping a verified mark of a retired owner; +/// `verified` also marks it, as a completed run does. +inline void tombstoneOwnerForTest(DB::Cas::Backend & backend, const String & owner_key, bool verified = false) +{ + OperationForTest op(backend); + const auto current = (*op).read(owner_key, DB::Cas::Retry::once()); + ASSERT_TRUE(current.has_value()); + const DB::Cas::OwnerObject owner = DB::Cas::decodeOwner(current->bytes); + const DB::Cas::OwnerObject tombstone = DB::Cas::nextOwnerLife( + owner, 1, verified || (owner.retirement_verified && owner.retired_at_ms.has_value())); + const auto put = (*op).replace(owner_key, DB::Cas::encodeOwner(tombstone), current->etag, DB::Cas::Retry::once()); + ASSERT_TRUE(std::holds_alternative(put)); +} + /// Stand in for the self-remount that `Pool::reportImpossibleInterference` schedules. That reaction /// trips the local write fence closed AND schedules a remount; a unit-test Pool runs no background /// remount (`background_watermark` is off by design there), so without this the fence stays closed and diff --git a/src/Disks/tests/gtest_ca_wiring.cpp b/src/Disks/tests/gtest_ca_wiring.cpp index 5489c2d070f49..2194ac8e80b72 100644 --- a/src/Disks/tests/gtest_ca_wiring.cpp +++ b/src/Disks/tests/gtest_ca_wiring.cpp @@ -354,6 +354,7 @@ namespace DB::ErrorCodes namespace DB::ContentAddressedSetting { extern const ContentAddressedSettingsString staging_backend; + extern const ContentAddressedSettingsUInt64 mount_lease_ttl_ms; } namespace @@ -3316,3 +3317,23 @@ TEST(CASWiringOps, RedriveOverEqualCommittedDestinationCompletesTheMove) EXPECT_TRUE(storage->existsDirectory(table + "X")); EXPECT_EQ(std::count_if(events.begin(), events.end(), [](const auto & e) { return e.type == DB::Cas::CasEventType::RefMove; }), 0); } + +/// A pool mounted from a 1-hour `cas_mount_lease_ttl_ms` declares the clamped 10 minutes in its slot. +TEST(CASWiringMount, OverlongConfiguredLeaseIsDeclaredClamped) +{ + auto settings = DB::Cas::tests::makeSettingsForTest( + "test", std::filesystem::temp_directory_path() / "ca_wiring_clamp_scratch"); + settings[DB::ContentAddressedSetting::mount_lease_ttl_ms] = 3'600'000; + settings.validate(); + auto storage = std::make_shared( + DB::Cas::tests::makeLocalObjectStorageForTest(), "pool", "srv1", "", nullptr, settings); + storage->startup(); + + const auto pool = storage->store(); + EXPECT_EQ(pool->poolConfig().mount_lease_ttl_ms.count(), 600'000); + DB::Cas::tests::OperationForTest op(*pool->poolBackendPtr()); + const auto mounts = DB::Cas::listMounts(*op, pool->layout(), /*now_ms=*/0, /*skew_margin_ms=*/0); + ASSERT_EQ(mounts.size(), 1u); + EXPECT_GT(mounts[0].lease.started_at_ms, 0u); + EXPECT_EQ(mounts[0].lease.expires_at_ms - mounts[0].lease.started_at_ms, 600'000u); +} diff --git a/src/Disks/tests/gtest_cas_backend.cpp b/src/Disks/tests/gtest_cas_backend.cpp index cb852249aa458..d178cccaa16e4 100644 --- a/src/Disks/tests/gtest_cas_backend.cpp +++ b/src/Disks/tests/gtest_cas_backend.cpp @@ -946,7 +946,7 @@ String readStorageObject(const DB::ObjectStoragePtr & storage, const String & ke } -TEST(CASObjectStorageBackend, PublishBlobStreamingUsesOrdinaryDefaultWriteTransport) +TEST(CASObjectStorageBackend, PublishBlobStreamingSettingsRequestSingleCreation) { auto storage = makePublicationRecordingStorage(); auto backend = std::make_shared(storage, ObjectStorageBackend::Mode::Native); @@ -964,7 +964,7 @@ TEST(CASObjectStorageBackend, PublishBlobStreamingUsesOrdinaryDefaultWriteTransp ASSERT_TRUE(storage->last_write_settings.has_value()); EXPECT_EQ(storage->last_write_settings->object_storage_request_mode, DB::ObjectStorageRequestMode::Default); EXPECT_EQ(storage->last_write_settings->object_storage_retry_profile, DB::ObjectStorageRetryProfile::Default); - EXPECT_EQ(storage->last_write_settings->s3_max_unexpected_write_error_retries_override, 0u); + EXPECT_TRUE(storage->last_write_settings->object_storage_single_attempt_object_creation); EXPECT_FALSE(storage->last_write_settings->s3_force_single_part_upload); EXPECT_TRUE(storage->last_write_settings->object_storage_write_if_none_match.empty()); EXPECT_TRUE(storage->last_write_settings->object_storage_write_if_match.empty()); @@ -1155,6 +1155,7 @@ TEST(CASObjectStorageBackend, PublishBlobCopiesStagedBytesWithNativeOnlyDefaultR EXPECT_EQ(storage->last_copy_settings->object_storage_copy_mode, DB::ObjectStorageCopyMode::NativeOnly); EXPECT_EQ(storage->last_copy_settings->object_storage_request_mode, DB::ObjectStorageRequestMode::Default); EXPECT_EQ(storage->last_copy_settings->object_storage_retry_profile, DB::ObjectStorageRetryProfile::Default); + EXPECT_TRUE(storage->last_copy_settings->object_storage_single_attempt_object_creation); EXPECT_TRUE(storage->last_copy_settings->object_storage_write_if_none_match.empty()); EXPECT_TRUE(storage->last_copy_settings->object_storage_write_if_match.empty()); EXPECT_EQ(readStorageObject(storage, destination), "staged-envelopepayload"); diff --git a/src/Disks/tests/gtest_cas_blob_publish_transport.cpp b/src/Disks/tests/gtest_cas_blob_publish_transport.cpp new file mode 100644 index 0000000000000..0b21e8c04f93f --- /dev/null +++ b/src/Disks/tests/gtest_cas_blob_publish_transport.cpp @@ -0,0 +1,287 @@ +#include + +#include "config.h" + +#if USE_AWS_S3 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/// Blob publication must put every object-creating request (`PutObject`, `CopyObject`, +/// `CompleteMultipartUpload`) on the wire at most once. These tests drive the backend's `publish` +/// against a counting S3 server whose client retries, as a disk's client does. + +namespace ProfileEvents +{ +extern const Event CASBlobPublishAttemptFailed; +} + +namespace DB::ErrorCodes +{ +extern const int S3_ERROR; +} + +using CountingS3::CountingS3Server; +using CountingS3::Verb; +using S3ObjectCreationTestStorage::forceOnePartMultipart; +using S3ObjectCreationTestStorage::makeRetryingStorage; +using S3ObjectCreationTestStorage::thrownCode; + +namespace +{ + +std::shared_ptr nativeBackend(DB::ObjectStoragePtr storage) +{ + return std::make_shared( + std::move(storage), DB::Cas::ObjectStorageBackend::Mode::Native, + /*single_attempt_control_plane_=*/true, /*attempt_timeout_ms_=*/5000, /*connect_timeout_cap_ms_=*/5000); +} + +DB::Cas::BlobPublishRequest streamingPublication() +{ + return DB::Cas::BlobPublishRequest{ + .destination_key = "blob", + .publication = DB::Cas::StreamingBlobPublication{ + .payload_size = 100, + .fresh_envelope = "envelope", + .open_payload = [] + { + return std::make_unique(std::string(100, 'p')); + }}}; +} + +/// One engine attempt of the backend's blob publication, so every counted request comes from below it. +void publishOnce(const DB::ObjectStoragePtr & storage, const DB::Cas::BlobPublishRequest & request) +{ + DB::Cas::CasRequests requests(DB::Cas::BackendPtr(nativeBackend(storage)), DB::Cas::Fence::open()); + auto op = requests.admit(); + op.publish(request, DB::Cas::Retry::once()); +} + +} + +/// A lost `CompleteMultipartUpload` is not resent by the transport. +TEST(CASBlobPublishTransport, LostCompleteIsOneWireRequest) +{ + (void)getContext(); + CountingS3Server server(CountingS3::dropping(Verb::CompleteMultipartUpload)); + auto storage = makeRetryingStorage(server.getUrl(), forceOnePartMultipart); + + EXPECT_EQ(thrownCode([&] { publishOnce(storage, streamingPublication()); }), DB::ErrorCodes::S3_ERROR); + EXPECT_EQ(server.count(Verb::CompleteMultipartUpload), 1u); +} + +namespace +{ + +/// The control plane in memory, the blob bodies on a real `S3ObjectStorage`: the writer's blob `HEAD` and +/// its publication go through the S3 transport, so the publication loop sees the transport's failures. +class S3BlobBodiesBackend final : public DB::Cas::InMemoryBackend +{ +public: + using InMemoryBackend::head; + + explicit S3BlobBodiesBackend(std::shared_ptr blob_bodies_) : blob_bodies(std::move(blob_bodies_)) {} + + std::optional head(const String & key, DB::Cas::TransportAccess & access) override + { + if (key.contains("/blobs/") && !key.ends_with(".meta")) + return blob_bodies->head(key, access); + return InMemoryBackend::head(key, access); + } + + void publish(const DB::Cas::BlobPublishRequest & request, DB::Cas::TransportAccess & access) override + { + blob_bodies->publish(request, access); + } + +private: + std::shared_ptr blob_bodies; +}; + +DB::Cas::PoolPtr openPool(const std::string & endpoint) +{ + auto backend = std::make_shared(nativeBackend(makeRetryingStorage(endpoint))); + auto pool = DB::Cas::Pool::open(backend, DB::Cas::PoolConfig{.pool_prefix = "p", .server_root_id = "test"}); + /// The publication loop paces its attempts on the engine's clock; a virtual one skips the backoff. + auto now = std::make_shared>(0); + pool->mountRequests().setNowFnForTest([now] { return now->load(); }); + pool->mountRequests().setSleepFnForTest([now](uint64_t ms) { now->fetch_add(ms + 1); }); + return pool; +} + +/// Stages a one-blob part for `payload` and returns its build, ready for `putBlob`. +DB::Cas::PartWriteTxnPtr startBuild(const DB::Cas::PoolPtr & pool, const std::string & payload) +{ + const DB::Cas::RootNamespace ns{"srv/tbl"}; + DB::Cas::PartWriteInfo info; + info.intended_ref = ns.string() + "/part"; + auto build = pool->beginPartWrite(info); + + DB::Cas::ManifestEntry entry; + entry.path = "a.bin"; + entry.placement = DB::Cas::EntryPlacement::Blob; + entry.ref = DB::Cas::tests::idOf(payload); + entry.blob_size = payload.size(); + build->precommitAdd(ns, "part", build->stageManifest({entry})); + return build; +} + +std::string s3Path(const std::string & key) +{ + return "/test-bucket/" + key; +} + +/// The requests of `verb` to `path`, in arrival order. +std::vector requestsTo(const CountingS3::ObjectStore & store, Verb verb, const std::string & path) +{ + std::vector matching; + for (auto & request : store.journal()) + if (request.verb == verb && request.path == path) + matching.push_back(std::move(request)); + return matching; +} + +/// Whether a `HEAD` of `path` arrived between the first and the second creating request to it. +bool observedBetweenCreations(const CountingS3::ObjectStore & store, const std::string & path) +{ + int creations = 0; + bool observed = false; + for (const auto & request : store.journal()) + { + if (request.path != path) + continue; + if (request.verb == Verb::PutObject || request.verb == Verb::CopyObject) + ++creations; + else if (request.verb == Verb::HeadObject && creations == 1) + observed = true; + } + return observed; +} + +/// The first request of `verb` reaches the store but is not applied, and its response is lost. +CountingS3::ObjectStore::FaultFn dropFirst(Verb verb) +{ + return [verb](const CountingS3::Request & request, Poco::Net::HTTPServerResponse &) + { + if (request.verb == verb && request.nth == 1) + throw CountingS3::DropResponse(); + return false; + }; +} + +CountingS3::ObjectStore::FaultFn dropFirstTwo(Verb verb) +{ + return [verb](const CountingS3::Request & request, Poco::Net::HTTPServerResponse &) + { + if (request.verb == verb && request.nth <= 2) + throw CountingS3::DropResponse(); + return false; + }; +} + +uint64_t failedPublicationAttempts() +{ + return ProfileEvents::global_counters[ProfileEvents::CASBlobPublishAttemptFailed]; +} + +} + +/// A lost `PUT` is not resent by the transport: the writer observes the key again and publishes a new +/// envelope, so a late landing of the first one can never carry the second one's `incarnation_tag`. +TEST(CASBlobPublishTransport, LostPutIsRepublishedWithAFreshEnvelope) +{ + (void)getContext(); + CountingS3::ObjectStore store(dropFirst(Verb::PutObject)); + CountingS3Server server(store.responder()); + auto pool = openPool(server.getUrl()); + const std::string payload = "composition-payload"; + auto build = startBuild(pool, payload); + const std::string path = s3Path(pool->layout().blobKey(DB::Cas::tests::idOf(payload))); + const uint64_t failed_before = failedPublicationAttempts(); + + DB::Cas::BlobSource source; + source.size = payload.size(); + source.open = [payload] { return std::make_unique(payload); }; + EXPECT_EQ(build->putBlob(DB::Cas::tests::idOf(payload), std::move(source)).size, payload.size()); + + const auto puts = requestsTo(store, Verb::PutObject, path); + ASSERT_EQ(puts.size(), 2u); + EXPECT_TRUE(observedBetweenCreations(store, path)) << "the second publication must follow a new observation"; + const uint64_t object_size = pool->poolMeta().blob_header_len + payload.size(); + const auto first = DB::Cas::decodeEnvelopeHeader(puts[0].body, object_size, DB::Cas::ObjectKind::Blob); + const auto second = DB::Cas::decodeEnvelopeHeader(puts[1].body, object_size, DB::Cas::ObjectKind::Blob); + EXPECT_TRUE(first.incarnation_tag != second.incarnation_tag) << "the second publication resent the first envelope"; + EXPECT_EQ(store.get(path), puts[1].body); + EXPECT_EQ(failedPublicationAttempts() - failed_before, 1u); +} + +/// A single lost attempt is routine and the next one usually succeeds, so only a repeated failure is +/// worth a client-visible warning. +TEST(CASBlobPublishTransport, FirstFailedAttemptIsBelowWarning) +{ + (void)getContext(); + CountingS3::ObjectStore store(dropFirstTwo(Verb::PutObject)); + CountingS3Server server(store.responder()); + auto pool = openPool(server.getUrl()); + const std::string payload = "two-lost-puts-payload"; + auto build = startBuild(pool, payload); + DB::Cas::tests::ScopedLogCapture capture("CasBlobPublishAttempt", "warning"); + + DB::Cas::BlobSource source; + source.size = payload.size(); + source.open = [payload] { return std::make_unique(payload); }; + build->putBlob(DB::Cas::tests::idOf(payload), std::move(source)); + + EXPECT_EQ(capture.count("failed, attempt 1 of"), 0u) << capture.captured(); + EXPECT_EQ(capture.count("failed, attempt 2 of"), 1u) << capture.captured(); +} + +/// A lost staged `CopyObject` is not resent: the staged copy is spent, so the retry streams. +TEST(CASBlobPublishTransport, LostStagedCopyIsRepublishedByStreaming) +{ + (void)getContext(); + CountingS3::ObjectStore store(dropFirst(Verb::CopyObject)); + CountingS3Server server(store.responder()); + auto pool = openPool(server.getUrl()); + const std::string payload = "staged-composition-payload"; + auto build = startBuild(pool, payload); + const std::string path = s3Path(pool->layout().blobKey(DB::Cas::tests::idOf(payload))); + const std::string staging_key = "p/staging/test/blob"; + const std::string staged_bytes = std::string(pool->poolMeta().blob_header_len, 'h') + payload; + store.put(s3Path(staging_key), staged_bytes); + + DB::Cas::BlobSource source; + source.size = payload.size(); + source.server_side_copy_from = staging_key; + source.open = [payload] { return std::make_unique(payload); }; + EXPECT_EQ(build->putBlob(DB::Cas::tests::idOf(payload), std::move(source)).size, payload.size()); + + EXPECT_EQ(requestsTo(store, Verb::CopyObject, path).size(), 1u); + EXPECT_EQ(requestsTo(store, Verb::PutObject, path).size(), 1u); + EXPECT_TRUE(observedBetweenCreations(store, path)) << "the streamed publication must follow a new observation"; + const auto published = store.get(path); + ASSERT_TRUE(published.has_value()); + EXPECT_NE(*published, staged_bytes); + EXPECT_EQ(published->substr(pool->poolMeta().blob_header_len), payload); +} + +#endif diff --git a/src/Disks/tests/gtest_cas_decommission.cpp b/src/Disks/tests/gtest_cas_decommission.cpp index 936e9655e9b14..155d3a24fe202 100644 --- a/src/Disks/tests/gtest_cas_decommission.cpp +++ b/src/Disks/tests/gtest_cas_decommission.cpp @@ -2,15 +2,34 @@ #include #include #include +#include #include +#include +#include +#include #include #include +#include +#include +#include +#include #include #include +#include #include +#include +#include +#include + +namespace ProfileEvents +{ + extern const Event CASRemountFailed; +} namespace DB::ErrorCodes { + extern const int ABORTED; + extern const int BAD_ARGUMENTS; extern const int CORRUPTED_DATA; } @@ -22,9 +41,27 @@ namespace { /// Open a store for the VICTIM srid over `backend` (the pool's future dead member). -PoolPtr openVictim(std::shared_ptr backend) +PoolPtr openVictim(BackendPtr backend) +{ + return Pool::open(std::move(backend), PoolConfig{.pool_prefix = "p", .server_root_id = "victim"}); +} + +OwnerObject readOwner(Backend & backend, const Layout & layout, const String & srid) +{ + OperationForTest op(backend); + const auto current = (*op).read(layout.ownerKey(srid), Retry::once()); + if (!current) + throw std::runtime_error("owner absent: " + layout.ownerKey(srid)); + return decodeOwner(current->bytes); +} + +MountLease readSlot(Backend & backend, const Layout & layout, const String & srid) { - return Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "victim"}); + OperationForTest op(backend); + const auto current = (*op).read(layout.mountKey(srid), Retry::once()); + if (!current) + throw std::runtime_error("mount absent: " + layout.mountKey(srid)); + return decodeMountLease(current->bytes); } void drainCompletedNamespaceRemovals(const std::shared_ptr & backend) @@ -40,27 +77,15 @@ void drainCompletedNamespaceRemovals(const std::shared_ptr & ba ASSERT_FALSE(runRegularRoundReclaiming(gc).deferred); } -/// Fails a delete for one or two designated keys -- either by throwing (a transient backend hiccup) or -/// by returning a synthetic `Mismatch` (a "listed but raced" outcome) -- delegating every other key to -/// the base `InMemoryBackend` untouched. Drives the drain phases' per-object fail-close path -/// (`deleteListedPrefix`/`sweepNamespace`, `CasDecommission.cpp`/`CasOrphanManifestSweep.cpp`): a -/// failure on one listed object must record a warning and let the rest of the sweep proceed, never -/// abort the whole phase. Injects on the `remove` PRIMITIVE, not the legacy `deleteExact`, so it -/// intercepts a caller on either surface. -/// -/// The thrown fault is a `Poco::TimeoutException`, the class the request engine classifies as a -/// transport failure and reissues (`Retry::standard()`); a `std::runtime_error` propagates immediately -/// and never exercises the retry path this test's own name claims to drive. `latch` keeps it armed -/// across every reissue of the SAME logical call, so the engine reaches its own retry deadline and -/// gives up rather than recovering on a later attempt -- a one-shot throw would be outlived by the -/// reissue and the delete would simply succeed. +/// Fails the delete of one key with a transient `Poco::TimeoutException` (the engine reissues it), and of +/// another with `Mismatch`. `latch` keeps the throw armed across every reissue, so the engine gives up. class FailingDeleteBackend : public InMemoryBackend { public: void failWithThrow(const String & key) { throw_key = key; } void failWithTokenMismatch(const String & key) { mismatch_key = key; } void latch() { latched = true; } - /// Clears every injected failure -- the resume half of a fail-then-retry test (Task 4). + /// Clears every injected failure. void disarm() { throw_key.clear(); mismatch_key.clear(); latched = false; } RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override @@ -178,237 +203,81 @@ std::vector> snapshotPrefixObjects( } } -/// Installs a same-UUID successor deterministically in the retirement tail's read/delete window. -/// Once armed, the backend recognizes the admin's clean farewell `putOverwrite`. On the next read of -/// either mutable control object it first captures the value that read observed, then bumps `epoch` -/// and reclaims `mount` with fresh tokens before returning the captured result. Thus the caller holds -/// exactly the stale token it would have obtained immediately before a concurrent restart reclaimed -/// the slot, without threads or sleeps. -class SuccessorReclaimAfterFarewellBackend : public InMemoryBackend +/// Once armed, swaps the owner to another uuid on the second owner read after the admin's slot claim +/// landed: the first is the claim recheck, the second is the tombstone's. Arm it after the victim closed, +/// or the victim's own lease writes count as the claim. +class SwapOwnerBeforeTombstoneBackend : public InMemoryBackend { public: - void armForSuccessorReclaim() { armed = true; } - - /// Injects on the `read` PRIMITIVE: the retirement tail's own reads of `mount_key`/`epoch_key` - /// (`CasDecommission.cpp`) go through `CasOperation::read`, not the legacy `get`. - std::optional read(const String & key, TransportAccess & access) override - { - std::optional result = InMemoryBackend::read(key, access); - if (farewell_seen && !successor_injected && (key == mount_key || key == epoch_key)) - injectSuccessor(access); - return result; - } + void arm() { armed = true; } + bool fired() const { return swapped; } - /// Injects on the `write` PRIMITIVE: `putOverwrite` is not one of the two verb-identity - /// exceptions (`putIfAbsent`/`casPut`), so both the legacy caller and `CasOperation::replace` - /// reach it here. std::expected write(const String & key, const String & bytes, const std::optional & expected_value, TransportAccess & access) override { const auto result = InMemoryBackend::write(key, bytes, expected_value, access); - if (armed && result.has_value() && key == mount_key) - { - const MountLease mount = decodeMountLease(bytes); - if (mount.min_active_build_sequence == std::numeric_limits::max()) - farewell_seen = true; - } + if (armed && result && key == mount_key && expected_value && !decodeMountLease(bytes).gc_fenced + && !decodeMountLease(bytes).isCleanRelease()) + claimed = true; return result; } - bool successorInjected() const { return successor_injected; } - const String & successorMountValue() const { return successor_mount_value; } - const String & successorEpochValue() const { return successor_epoch_value; } - const String & successorMountBytes() const { return successor_mount_bytes; } - const String & successorEpochBytes() const { return successor_epoch_bytes; } - -private: - /// Every request here is issued on the PRIMITIVE `write`, using the `access` token the caller's - /// own in-flight request already holds -- no `CasRequests`/`CasOperation` exists inside this - /// reentrant hook to mint one. The captured `successor_*_value` fields are the raw wire values - /// `write` returned, which the tests compare against a real incarnation's rendered value - /// (`PersistedEtag::capture`) taken through their own `CasOperation`. - void injectSuccessor(TransportAccess & access) - { - const auto epoch = InMemoryBackend::read(epoch_key, access); - const auto mount = InMemoryBackend::read(mount_key, access); - if (!epoch || !mount) - throw std::runtime_error("successor-reclaim fixture: control object disappeared before reclaim"); - - ServerEpoch epoch_value = decodeServerEpoch(epoch->bytes); - const uint64_t successor_writer_epoch = epoch_value.next_writer_epoch; - ++epoch_value.next_writer_epoch; - successor_epoch_bytes = encodeServerEpoch(epoch_value); - const auto epoch_written = InMemoryBackend::write(epoch_key, successor_epoch_bytes, epoch->value, access); - if (!epoch_written) - throw std::runtime_error("successor-reclaim fixture: epoch bump conflicted"); - successor_epoch_value = *epoch_written; - - MountLease mount_value = decodeMountLease(mount->bytes); - mount_value.writer_epoch = successor_writer_epoch; - ++mount_value.seq; - ++mount_value.started_at_ms; - mount_value.expires_at_ms = mount_value.started_at_ms + 30'000; - mount_value.min_active_build_sequence = 0; - mount_value.gc_fenced = false; - successor_mount_bytes = encodeMountLease(mount_value); - const auto mount_written = InMemoryBackend::write(mount_key, successor_mount_bytes, mount->value, access); - if (!mount_written) - throw std::runtime_error("successor-reclaim fixture: mount reclaim conflicted"); - successor_mount_value = *mount_written; - successor_injected = true; - } - - inline static const String mount_key = "p/gc/server-roots/victim/mount"; - inline static const String epoch_key = "p/gc/server-roots/victim/epoch"; - bool armed = false; - bool farewell_seen = false; - bool successor_injected = false; - String successor_mount_value; - String successor_epoch_value; - String successor_mount_bytes; - String successor_epoch_bytes; -}; - -/// Recreates the mutable slot objects immediately after decommission successfully deletes `epoch`. -/// This models a same-UUID successor starting in the final retirement window: `owner` remains the -/// unchanged identity anchor, while the successor legitimately creates a fresh `epoch` and `mount`. -class SuccessorReclaimAfterEpochDeleteBackend : public InMemoryBackend -{ -public: - void armForSuccessorReclaim() { armed = true; } - - /// Injects on the `remove` PRIMITIVE: `deleteSlotObject`'s epoch delete (`CasDecommission.cpp`) - /// goes through `CasOperation::remove`, not the legacy `deleteExact`. - RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override + std::optional read(const String & key, TransportAccess & access) override { - const RawRemoval result = InMemoryBackend::remove(key, expected_value, access); - if (armed && !successor_injected && key == epoch_key && result == RawRemoval::Removed) - injectSuccessor(access); - return result; + if (claimed && !swapped && key == owner_key && ++owner_reads == 2) + { + const auto current = InMemoryBackend::read(key, access); + OwnerObject foreign = nextOwnerLife(decodeOwner(current->bytes), std::nullopt, false); + foreign.server_uuid = UInt128(0xBEEF); + if (!InMemoryBackend::write(key, encodeOwner(foreign), current->value, access)) + throw std::runtime_error("owner swap fixture: the rewrite conflicted"); + swapped = true; + } + return InMemoryBackend::read(key, access); } - bool successorInjected() const { return successor_injected; } - uint64_t ownerRewriteAttempts() const { return owner_rewrite_attempts; } - const String & successorMountValue() const { return successor_mount_value; } - const String & successorEpochValue() const { return successor_epoch_value; } - const String & successorMountBytes() const { return successor_mount_bytes; } - const String & successorEpochBytes() const { return successor_epoch_bytes; } - private: - /// Counts on the `write` PRIMITIVE: the owner tombstone write this test asserts is never - /// attempted (`CasDecommission.cpp`'s `op.replace`) reaches the store through here, not through - /// the legacy `putOverwrite`. Guarded on `expected_value`: the primitive also sees the owner - /// anchor's own CREATE during this fixture's `openVictim`, which `putOverwrite` (a conditional - /// REPLACE only) never did -- counting it would start this counter at 1 before the interesting - /// part of the test begins. - std::expected write(const String & key, const String & bytes, - const std::optional & expected_value, TransportAccess & access) override - { - if (key == owner_key && expected_value) - ++owner_rewrite_attempts; - return InMemoryBackend::write(key, bytes, expected_value, access); - } - - /// On the `write` PRIMITIVE, using the caller's own in-flight `access`: this hook is reentrant - /// (called from inside another primitive override on the same object), so it reaches the store - /// directly through `InMemoryBackend::write` rather than admitting a fresh request of its own. - void injectSuccessor(TransportAccess & access) - { - successor_epoch_bytes = encodeServerEpoch(ServerEpoch{.next_writer_epoch = 102}); - const auto epoch_written = InMemoryBackend::write(epoch_key, successor_epoch_bytes, std::nullopt, access); - if (!epoch_written) - throw std::runtime_error("late-successor fixture: epoch recreation conflicted"); - successor_epoch_value = *epoch_written; - - successor_mount_bytes = encodeMountLease(MountLease{ - .server_uuid = UInt128(0x1234), - .writer_epoch = 101, - .hostname = "successor", - .pid = 42, - .started_at_ms = 1'000, - .seq = 1, - .expires_at_ms = 31'000, - .min_active_build_sequence = 0, - }); - const auto mount_written = InMemoryBackend::write(mount_key, successor_mount_bytes, std::nullopt, access); - if (!mount_written) - throw std::runtime_error("late-successor fixture: mount recreation conflicted"); - successor_mount_value = *mount_written; - successor_injected = true; - } - inline static const String mount_key = "p/gc/server-roots/victim/mount"; - inline static const String epoch_key = "p/gc/server-roots/victim/epoch"; inline static const String owner_key = "p/gc/server-roots/victim/owner"; bool armed = false; - bool successor_injected = false; - uint64_t owner_rewrite_attempts = 0; - String successor_mount_value; - String successor_epoch_value; - String successor_mount_bytes; - String successor_epoch_bytes; + bool claimed = false; + bool swapped = false; + int owner_reads = 0; }; -/// Rewrites the owner anchor after decommission reads it but before its conditional tombstone write. -/// Returning the captured result gives decommission a stale owner token, deterministically modeling -/// the successor race without threads or sleeps. -class SuccessorOwnerRewriteBeforeTombstoneBackend : public InMemoryBackend +/// Once armed, fences the admin's body right before its farewell lands, so the farewell conflicts and +/// `MountLeaseRenewer::terminate` returns without an outcome. Arm it after the victim closed, or the +/// victim's own farewell is the one fenced. +class FenceBeforeFarewellBackend : public InMemoryBackend { public: - void armForSuccessorRewrite() { armed = true; } + void arm() { armed = true; } + bool fired() const { return fenced; } - /// Injects on the `read` PRIMITIVE: the tombstone tail's owner read (`CasDecommission.cpp`) - /// goes through `CasOperation::read`, not the legacy `get`. - std::optional read(const String & key, TransportAccess & access) override + std::expected write(const String & key, const String & bytes, + const std::optional & expected_value, TransportAccess & access) override { - std::optional result = InMemoryBackend::read(key, access); - if (armed && epoch_deleted && !successor_injected && key == owner_key && result) + if (armed && !fenced && key == mount_key && expected_value && decodeMountLease(bytes).isCleanRelease()) { - successor_owner_bytes = encodeOwner(OwnerObject{ - .server_uuid = decodeOwner(result->bytes).server_uuid, - .retired_at_ms = std::nullopt, - }); - const auto put = InMemoryBackend::write(owner_key, successor_owner_bytes, result->value, access); - if (!put.has_value()) - throw std::runtime_error("owner-successor fixture: owner rewrite conflicted"); - successor_owner_value = *put; - successor_injected = true; + const auto current = InMemoryBackend::read(key, access); + if (!InMemoryBackend::write(key, encodeMountLease(decodeMountLease(current->bytes).fencedCopy()), current->value, access)) + throw std::runtime_error("fence-before-farewell fixture: the fence conflicted"); + fenced = true; } - return result; - } - - /// Injects on the `remove` PRIMITIVE: `deleteSlotObject`'s epoch delete (`CasDecommission.cpp`) - /// goes through `CasOperation::remove`, not the legacy `deleteExact`. - RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override - { - const RawRemoval result = InMemoryBackend::remove(key, expected_value, access); - if (armed && key == epoch_key && result == RawRemoval::Removed) - epoch_deleted = true; - return result; + return InMemoryBackend::write(key, bytes, expected_value, access); } - bool successorInjected() const { return successor_injected; } - const String & successorOwnerValue() const { return successor_owner_value; } - const String & successorOwnerBytes() const { return successor_owner_bytes; } - private: - inline static const String epoch_key = "p/gc/server-roots/victim/epoch"; - inline static const String owner_key = "p/gc/server-roots/victim/owner"; + inline static const String mount_key = "p/gc/server-roots/victim/mount"; bool armed = false; - bool epoch_deleted = false; - bool successor_injected = false; - String successor_owner_value; - String successor_owner_bytes; + bool fenced = false; }; -/// Seed one victim table with `committed` committed refs and `precommits` dangling precommit bindings, -/// via the raw ref-log seeding helpers (fixture idiom of e.g. `gtest_cas_gc_fold.cpp`: `writeManifestRaw` -/// + `publishCommittedTransition`/`addPrecommitTransition` against `victim`'s own backend/layout) -- this -/// fixture only needs the ref-table SHAPE `dropNamespace` erases, not a real build. Precommit bindings -/// are seeded at an artificially high `writer_epoch` so the writer's own stale-precommit sweep (armed -/// unconditionally by this table's recovery, unrelated to decommission -- spec §Clean Up Old Precommits) -/// never reclaims them, in its OWN separate transaction, ahead of `dropNamespace`'s removal. +/// Seed one victim table with `committed` committed refs and `precommits` dangling precommit bindings through +/// the raw ref-log helpers: only the ref-table shape `dropNamespace` erases, not a real build. Precommits +/// carry a high `writer_epoch`, so the writer's stale-precommit sweep never reclaims them in its own +/// transaction ahead of `dropNamespace`. void makeTableWithRefs(Pool & victim, const String & ns_str, uint64_t committed, uint64_t precommits) { const RootNamespace ns(ns_str); @@ -464,37 +333,131 @@ void makeTableWithRefs(Pool & victim, const String & ns_str, uint64_t committed, ASSERT_EQ(victim.listRefs(ns).size(), committed); } -/// Pre-precommit manifest debris: a staged manifest body under `ns_str`, at the store's own -/// `writer_epoch`, named by NO owner event -- a build the writer staged and never finished (fixture -/// idiom of `gtest_cas_orphan_manifest_sweep.cpp`'s `EligibleAndUnownedIsDeleted`). `build_sequence = 99` -/// is picked well clear of `makeTableWithRefs`'s own committed/precommit build sequences so it can never -/// collide with a real owned manifest key. Returns the seeded body's `ManifestId` so a caller can target -/// it (e.g. its exact object key) for further fixture setup. +/// Manifest debris: a staged manifest body under `ns_str` at the store's own `writer_epoch` that no owner +/// event names, a build staged and never finished. `build_sequence = 99` stays clear of `makeTableWithRefs`. ManifestId seedOrphanManifestBody(Pool & victim, const String & ns_str) { const RootNamespace ns(ns_str); const ManifestRef ref{.writer_epoch = victim.writerEpoch(), .build_sequence = 99, .manifest_ordinal = 1}; const ManifestId id = writeManifestRaw(*victim.poolBackendPtr(), victim.layout(), ns, ref, {}); - /// EXPECT, not ASSERT: this function returns a value now, and ASSERT_* expands to a bare `return;` - /// -- invalid in a non-void function. OperationForTest op(*victim.poolBackendPtr()); EXPECT_TRUE((*op).head(victim.layout().manifestKey(id), Retry::once()).has_value()); return id; } -/// THE MANIFEST-DEBRIS DRAIN NO LONGER DELETES, AND THE FIXTURES BELOW SAY SO RATHER THAN WORKING -/// AROUND IT. The drain goes through `sweepNamespace`, which is subject to the §6 deletion premise: a -/// manifest of an epoch-`E` build is deletable only once the namespace's sealed fold cursor sits in an -/// epoch STRICTLY above `E`. Every object in these fixtures -- the table's ref stream, the debris, and -/// the removal transaction decommission itself appends -- lives in ONE writer epoch, and a single-epoch -/// pool cannot satisfy that: any cursor high enough to clear the debris's epoch also sits above the -/// removal record, which would strip the tail-removal protection off the table's real manifests. The -/// two facts are mutually exclusive here, so there is no honest seeding that restores the deletions; -/// the tests assert retention, and the drain's reclaim path returns with registers R2/R3 (Stage B). +/// The manifest-debris drain deletes an epoch-`E` manifest only once the namespace's sealed fold cursor sits +/// above `E`. In these single-epoch fixtures no cursor can clear the debris without also clearing the removal +/// record, so the tests assert retention. } -TEST(CASDecommission, RefusesLiveMember) + +/// The pool "p" with the member "victim" every test decommissions; the helpers work on `victim_backend`. +class CASDecommission : public ::testing::Test +{ +protected: + const Layout layout{"p"}; + BackendPtr victim_backend; + + /// Opens the victim over `backend`, runs `prepare` on it and closes it; `backend` then backs every helper + /// below. + template + std::shared_ptr startVictim(std::shared_ptr backend = std::make_shared(), const std::function & prepare = {}) + { + victim_backend = backend; + auto victim = openVictim(backend); + if (prepare) + prepare(*victim); + return backend; + } + + static void oneTable(Pool & victim) { makeTableWithRefs(victim, "victim/db/t1", 1, 0); } + /// `oneTable` plus a staged manifest no ref names; returns the manifest's key. + static String oneTableWithDebris(Pool & victim) + { + oneTable(victim); + return victim.layout().manifestKey(seedOrphanManifestBody(victim, "victim/db/t1")); + } + + /// The owner, epoch and mount objects, bytes and incarnation. + std::vector>> controlObjects() const + { + OperationForTest op(*victim_backend); + std::vector>> objects; + for (const String & key : {layout.ownerKey("victim"), layout.epochKey("victim"), layout.mountKey("victim")}) + { + const auto got = (*op).read(key, Retry::once()); + objects.push_back(got ? std::optional(std::pair(got->bytes, got->etag)) : std::nullopt); + } + return objects; + } + + DecommissionReport run(const String & admin, PoolConfig config = {}) + { + config.pool_prefix = "p"; + config.server_root_id = admin; + return decommissionPoolMember(victim_backend, std::move(config), "victim"); + } + + /// The run's clock and retry sleep, and the admin's lease, on `clock`. The closures own the clock: the + /// admin's renewer may outlive the call that installed them. + DecommissionReport runOnClock(const String & admin, const std::shared_ptr & clock, PoolConfig config = {}) + { + config.pool_prefix = "p"; + config.server_root_id = admin; + return decommissionPoolMember(victim_backend, std::move(config), "victim", /*sink=*/{}, /*request_gc_round=*/{}, + [clock] { return clock->nowFn()(); }, [clock](uint64_t ms) { clock->sleepFn()(ms); }); + } + + OwnerObject owner() const { return readOwner(*victim_backend, layout, "victim"); } + MountLease slot() const { return readSlot(*victim_backend, layout, "victim"); } + + /// The owner fields a decommission decides: its life, whether it is retired, whether the retirement is verified. + void expectOwner(uint64_t life, bool retired, bool verified) const + { + const OwnerObject current = owner(); + EXPECT_EQ(current.owner_life, life); + EXPECT_EQ(current.retired_at_ms.has_value(), retired); + EXPECT_EQ(current.retirement_verified, verified); + } + + void seed(const String & key) const + { + OperationForTest op(*victim_backend); + ASSERT_TRUE(std::holds_alternative((*op).create(key, "x", Retry::once()))); + } + + bool exists(const String & key) const + { + OperationForTest op(*victim_backend); + return (*op).head(key, Retry::once()).has_value(); + } + + /// A fake clock far past any real host's boot time, so a split between it and the real boot clock fails. + static std::shared_ptr farClock() + { + auto clock = std::make_shared(); + clock->now = 1'000'000'000'000'000ULL; + return clock; + } + + /// A pool on real time with a 1 s lease renewed every 200 ms. + static PoolConfig realTimeConfig() + { + return PoolConfig{ + .pool_prefix = "p", + .server_root_id = "", + .mount_lease_ttl_ms = std::chrono::milliseconds(1000), + .mount_renew_period = std::chrono::milliseconds(200), + .cas_request_budget = CasRequestBudget{ + .attempt_timeout_ms = 100, .lease_safety_margin_ms = 100, .connect_timeout_cap_ms = std::nullopt}}; + } + + void fenceSlot() const { fenceOutMountForTest(*victim_backend, layout.mountKey("victim")); } + void tombstoneOwner() const { tombstoneOwnerForTest(*victim_backend, layout.ownerKey("victim")); } +}; + +TEST_F(CASDecommission, RefusesLiveMember) { auto backend = std::make_shared(); auto victim = openVictim(backend); /// keeps its mount lease unexpired — the member is alive @@ -505,7 +468,30 @@ TEST(CASDecommission, RefusesLiveMember) }); } -TEST(CASDecommission, ClaimsDeadMemberAndBumpsEpoch) +/// The refusal names the state a hard-killed member has to reach. `expired` in `system.cas_mounts` is +/// wall-clock arithmetic; the claim trusts only a terminal slot (`terminated` or `fenced`), and an +/// expired, unfenced lease takes exactly the branch an unexpired one takes here. +TEST_F(CASDecommission, NoWaitRefusalNamesTheTerminalStates) +{ + auto backend = std::make_shared(); + auto victim = openVictim(backend); + + try + { + Pool::openForDecommission(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + FAIL() << "expected the live member to be refused"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ABORTED); + EXPECT_NE(e.message().find("alive or contended"), String::npos) << e.message(); + EXPECT_NE(e.message().find("state = 'terminated'"), String::npos) << e.message(); + EXPECT_NE(e.message().find("state = 'fenced'"), String::npos) << e.message(); + EXPECT_NE(e.message().find("GC round"), String::npos) << e.message(); + } +} + +TEST_F(CASDecommission, ClaimsDeadMemberAndBumpsEpoch) { auto backend = std::make_shared(); uint64_t victim_epoch = 0; @@ -521,7 +507,7 @@ TEST(CASDecommission, ClaimsDeadMemberAndBumpsEpoch) EXPECT_EQ(admin->poolConfig().server_root_id, "victim"); } -TEST(CASDecommission, AlwaysRenewsAdminClaimEvenWhenHostDiskIsObserveOnly) +TEST_F(CASDecommission, AlwaysRenewsAdminClaimEvenWhenHostDiskIsObserveOnly) { auto backend = std::make_shared(); { @@ -536,9 +522,22 @@ TEST(CASDecommission, AlwaysRenewsAdminClaimEvenWhenHostDiskIsObserveOnly) backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin", .background_watermark = false}, "victim"); ASSERT_TRUE(admin != nullptr); EXPECT_TRUE(admin->poolConfig().background_watermark); + EXPECT_TRUE(admin->poolConfig().bounded_background_renewal); +} + +TEST_F(CASDecommission, OpensOverARetiredOwner) +{ + auto backend = startVictim(); + tombstoneOwner(); + auto admin = Pool::openForDecommission(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + ASSERT_TRUE(admin != nullptr); + /// The claim only reads: the owner stays retired at the life it had, and the admin holds the slot. + EXPECT_TRUE(owner().retired_at_ms.has_value()); + EXPECT_EQ(owner().owner_life, 2u); + EXPECT_EQ(slot().writer_epoch, admin->liveWriterEpoch()); } -TEST(CASDecommission, RefusesUnknownMember) +TEST_F(CASDecommission, RefusesUnknownMember) { auto backend = std::make_shared(); expectThrowsCode(ErrorCodes::BAD_ARGUMENTS, [&] @@ -547,10 +546,34 @@ TEST(CASDecommission, RefusesUnknownMember) }); } -TEST(CASDecommission, SecondConcurrentDecommissionRefused) +/// `SYSTEM CAS DROP POOL MEMBER 'a/b'` is refused before it touches the member, even when `a/b` owns its +/// owner object: the drain selects objects by key prefix, so it is defined only for ids without `/`. +TEST_F(CASDecommission, RefusesIdWithSlash) { auto backend = std::make_shared(); - { auto victim = openVictim(backend); } + const Layout layout("p"); + DB::Cas::tests::seedPoolMetaForRestart(*backend); + OperationForTest raw_op(*backend); + ASSERT_TRUE(std::holds_alternative((*raw_op).create(layout.ownerKey("shard-01/replica-a"), + encodeOwner(OwnerObject{.server_uuid = UInt128(1), .retired_at_ms = std::nullopt}), Retry::once()))); + const auto owner_before = (*raw_op).read(layout.ownerKey("shard-01/replica-a"), Retry::once()); + ASSERT_TRUE(owner_before.has_value()); + + DB::Cas::tests::expectThrowsCodeWithMessage(ErrorCodes::BAD_ARGUMENTS, "contains '/'", [&] + { + decommissionPoolMember(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "shard-01/replica-a"); + }); + + const auto owner_after = (*raw_op).read(layout.ownerKey("shard-01/replica-a"), Retry::once()); + ASSERT_TRUE(owner_after.has_value()); + EXPECT_EQ(owner_after->etag, owner_before->etag); + EXPECT_FALSE((*raw_op).head(layout.mountKey("shard-01/replica-a"), Retry::once()).has_value()); + EXPECT_FALSE((*raw_op).head(layout.epochKey("shard-01/replica-a"), Retry::once()).has_value()); +} + +TEST_F(CASDecommission, SecondConcurrentDecommissionRefused) +{ + auto backend = startVictim(); auto first = Pool::openForDecommission(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); expectThrowsCode(ErrorCodes::ABORTED, [&] @@ -559,11 +582,9 @@ TEST(CASDecommission, SecondConcurrentDecommissionRefused) }); } -TEST(CASDecommission, DuplicateLifeIdRefusesBeforeAnyNamespaceOrSlotMutation) +TEST_F(CASDecommission, DuplicateLifeIdRefusesBeforeAnyNamespaceOrSlotMutation) { - auto backend = std::make_shared(); - { auto victim = openVictim(backend); } - const Layout layout("p"); + auto backend = startVictim(); RefCatalog catalog; catalog.entries = { CatalogEntry{.ns = RootNamespace{"victim/a"}, .state = NsState::Live, .incarnation = UInt128{77}}, @@ -578,34 +599,17 @@ TEST(CASDecommission, DuplicateLifeIdRefusesBeforeAnyNamespaceOrSlotMutation) ASSERT_TRUE(empty_catalog); ASSERT_TRUE(std::holds_alternative((*raw_op).replace( layout.refCatalogKey(), encodeRefCatalog(catalog), empty_catalog->etag, Retry::once()))); - const auto owner_before = (*raw_op).read(layout.ownerKey("victim"), Retry::once()); - const auto epoch_before = (*raw_op).read(layout.epochKey("victim"), Retry::once()); - const auto mount_before = (*raw_op).read(layout.mountKey("victim"), Retry::once()); - ASSERT_TRUE(owner_before); - ASSERT_TRUE(epoch_before); - ASSERT_TRUE(mount_before); - - EXPECT_THROW(decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"), DB::Exception); - const auto owner_after = (*raw_op).read(layout.ownerKey("victim"), Retry::once()); - const auto epoch_after = (*raw_op).read(layout.epochKey("victim"), Retry::once()); - const auto mount_after = (*raw_op).read(layout.mountKey("victim"), Retry::once()); - ASSERT_TRUE(owner_after); - ASSERT_TRUE(epoch_after); - ASSERT_TRUE(mount_after); - EXPECT_EQ(owner_after->bytes, owner_before->bytes); - EXPECT_EQ(owner_after->etag, owner_before->etag); - EXPECT_EQ(epoch_after->bytes, epoch_before->bytes); - EXPECT_EQ(epoch_after->etag, epoch_before->etag); - EXPECT_EQ(mount_after->bytes, mount_before->bytes); - EXPECT_EQ(mount_after->etag, mount_before->etag); + const auto before = controlObjects(); + + EXPECT_THROW(run("admin"), DB::Exception); + EXPECT_EQ(controlObjects(), before); } -TEST(CASDecommission, CatalogCutIsValidatedBeforeImpersonationAndReusedForSelection) +TEST_F(CASDecommission, CatalogCutIsValidatedBeforeImpersonationAndReusedForSelection) { auto backend = std::make_shared(); + victim_backend = backend; { auto victim = Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "victim"}); } - const Layout layout("p"); RefCatalog ambiguous; ambiguous.entries = { @@ -617,38 +621,20 @@ TEST(CASDecommission, CatalogCutIsValidatedBeforeImpersonationAndReusedForSelect .removal_started_round = 1}, }; - OperationForTest raw_op(*backend); - const auto owner_before = (*raw_op).read(layout.ownerKey("victim"), Retry::once()); - const auto epoch_before = (*raw_op).read(layout.epochKey("victim"), Retry::once()); - const auto mount_before = (*raw_op).read(layout.mountKey("victim"), Retry::once()); - ASSERT_TRUE(owner_before); - ASSERT_TRUE(epoch_before); - ASSERT_TRUE(mount_before); + const auto before = controlObjects(); backend->armCatalogReplacement(layout.refCatalogKey(), std::move(ambiguous)); - EXPECT_THROW(decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"), DB::Exception); + EXPECT_THROW(run("admin"), DB::Exception); ASSERT_TRUE(backend->fired()); - const auto owner_after = (*raw_op).read(layout.ownerKey("victim"), Retry::once()); - const auto epoch_after = (*raw_op).read(layout.epochKey("victim"), Retry::once()); - const auto mount_after = (*raw_op).read(layout.mountKey("victim"), Retry::once()); - ASSERT_TRUE(owner_after); - ASSERT_TRUE(epoch_after); - ASSERT_TRUE(mount_after); - EXPECT_EQ(owner_after->bytes, owner_before->bytes); - EXPECT_EQ(owner_after->etag, owner_before->etag); - EXPECT_EQ(epoch_after->bytes, epoch_before->bytes); - EXPECT_EQ(epoch_after->etag, epoch_before->etag); - EXPECT_EQ(mount_after->bytes, mount_before->bytes); - EXPECT_EQ(mount_after->etag, mount_before->etag); + EXPECT_EQ(controlObjects(), before); } -TEST(CASDecommission, NamespaceSelectionUsesThePreImpersonationCut) +TEST_F(CASDecommission, NamespaceSelectionUsesThePreImpersonationCut) { auto backend = std::make_shared(); + victim_backend = backend; { auto victim = Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "victim"}); } - const Layout layout("p"); RefCatalog later; later.entries = { @@ -656,19 +642,18 @@ TEST(CASDecommission, NamespaceSelectionUsesThePreImpersonationCut) }; backend->armCatalogReplacement(layout.refCatalogKey(), std::move(later)); - const DecommissionReport report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + const DecommissionReport report = run("admin"); ASSERT_TRUE(backend->fired()); EXPECT_EQ(report.namespaces_removed, 0u) << "a namespace visible only to mount safety's later observation is outside the validated cut"; EXPECT_EQ(report.namespaces_already_removed, 0u); } -TEST(CASDecommission, SameNameRebirthAfterTheCutIsRefusedWithoutTouchingTheNewLife) +TEST_F(CASDecommission, SameNameRebirthAfterTheCutIsRefusedWithoutTouchingTheNewLife) { auto backend = std::make_shared(); + victim_backend = backend; { auto victim = Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "victim"}); } - const Layout layout("p"); const RootNamespace ns{"victim/same"}; const NamespaceLifeId old_life = NamespaceLifeId::fromCatalogEntry(ns, UInt128{70}); const NamespaceLifeId new_life = NamespaceLifeId::fromCatalogEntry(ns, UInt128{71}); @@ -712,8 +697,7 @@ TEST(CASDecommission, SameNameRebirthAfterTheCutIsRefusedWithoutTouchingTheNewLi String refusal; try { - (void)decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + (void)run("admin"); } catch (const DB::Exception & e) { @@ -726,10 +710,9 @@ TEST(CASDecommission, SameNameRebirthAfterTheCutIsRefusedWithoutTouchingTheNewLi << "decommission must not append a removal transaction to the post-cut incarnation"; } -TEST(CASDecommission, VictimNameMatchesOneCanonicalPathComponent) +TEST_F(CASDecommission, VictimNameMatchesOneCanonicalPathComponent) { - auto backend = std::make_shared(); - { auto victim = openVictim(backend); } + auto backend = startVictim(); const RootNamespace neighbor_ns{"victim2/db/t1"}; { @@ -737,8 +720,7 @@ TEST(CASDecommission, VictimNameMatchesOneCanonicalPathComponent) makeTableWithRefs(*neighbor, neighbor_ns.string(), /*committed=*/1, /*precommits=*/0); } - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + const auto report = run("admin"); EXPECT_EQ(report.namespaces_removed, 0u); auto neighbor = Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "victim2"}); @@ -746,9 +728,10 @@ TEST(CASDecommission, VictimNameMatchesOneCanonicalPathComponent) << "decommissioning victim must not select victim2 by raw string prefix"; } -TEST(CASDecommission, ErasesAllVictimNamespaces) +TEST_F(CASDecommission, ErasesAllVictimNamespaces) { auto backend = std::make_shared(); + victim_backend = backend; { auto victim = openVictim(backend); /// Two tables: ns "victim/db/t1" with 2 committed refs, ns "victim/db/t2" with 1 committed @@ -757,8 +740,7 @@ TEST(CASDecommission, ErasesAllVictimNamespaces) makeTableWithRefs(*victim, "victim/db/t2", /*committed=*/1, /*precommits=*/1); } - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + const auto report = run("admin"); EXPECT_EQ(report.srid, "victim"); EXPECT_EQ(report.namespaces_removed, 2u); @@ -767,48 +749,34 @@ TEST(CASDecommission, ErasesAllVictimNamespaces) EXPECT_EQ(report.precommits_removed, 1u); EXPECT_EQ(report.edge_deltas_emitted, 4u); - /// Terminal publication is writer work; exact catalog-row deletion remains GC work. The first - /// command therefore keeps the slot as an ownership anchor, and a retry may retire it only after - /// GC's next invocation drains the completed `Removing` rows. - EXPECT_FALSE(report.warnings.empty()); - EXPECT_FALSE(report.slot_removed); + /// Terminal publication is writer work; exact catalog-row deletion remains GC work, reported as + /// pending rather than as a failure. + EXPECT_TRUE(report.warnings.empty()); + EXPECT_TRUE(report.member_retired); + EXPECT_EQ(report.namespaces_pending_gc, 2u); drainCompletedNamespaceRemovals(backend); - const auto retired = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin2"}, "victim"); - EXPECT_TRUE(retired.warnings.empty()); - EXPECT_TRUE(retired.slot_removed); -} - -/// Task 2 review finding 1: `makeTableWithRefs`'s precommit seed uses an artificially high -/// `writer_epoch` (999999) specifically to dodge the writer's OWN stale-precommit sweep -- which -/// means it never exercised the path a REAL victim precommit takes. A genuine writer stamps -/// `manifest_ref.writer_epoch` from its OWN `liveWriterEpoch()` at precommit time -/// (`PartWriteTxn::precommitAdd`, CasPool.cpp:2087), i.e. the victim's era -- always LOWER than the admin -/// mount's freshly-minted epoch (`openForDecommission` always bumps strictly higher). `appendRefOps` -/// hoists `maybeSweepStalePrecommits` at its top (CasPool.cpp:1716), so without the -/// `skip_stale_precommit_sweep` fix that sweep would reclaim this realistic-epoch precommit in its -/// OWN transaction before `dropNamespace`'s removal transaction ever counts it, leaving -/// `precommits_removed` at 0 for exactly the case that matters. -TEST(CASDecommission, CountsRealisticEpochPrecommit) + const auto rerun = run("admin2"); + EXPECT_TRUE(rerun.warnings.empty()); + EXPECT_TRUE(rerun.member_retired); + EXPECT_EQ(rerun.namespaces_pending_gc, 0u); + EXPECT_EQ(rerun.namespaces_already_removed, 0u); +} + +/// A real precommit carries the victim's own epoch, below the admin's. The writer's stale-precommit sweep +/// must not reclaim it in its own transaction before the namespace drop counts it. +TEST_F(CASDecommission, CountsRealisticEpochPrecommit) { - auto backend = std::make_shared(); - uint64_t victim_epoch = 0; + startVictim(std::make_shared(), [](Pool & victim) { - auto victim = openVictim(backend); - victim_epoch = victim->writerEpoch(); - makeTableWithRefs(*victim, "victim/db/t1", /*committed=*/1, /*precommits=*/0); - + oneTable(victim); const RootNamespace ns("victim/db/t1"); - /// `build_sequence = 2`: distinct from `makeTableWithRefs`'s committed ref (`build_sequence = 1`) - /// -- a REAL build's `ManifestRef` is unique per build, and a colliding one would trip the ref - /// state machine's "manifest already has a conflicting owner" guard. - const ManifestRef ref{.writer_epoch = victim_epoch, .build_sequence = 2, .manifest_ordinal = 1}; - writeManifestRaw(*victim->poolBackendPtr(), victim->layout(), ns, ref, {}); - addPrecommitTransition(*victim->poolBackendPtr(), victim->layout(), ns, UInt128(1), "precommit_0", std::nullopt, ref); - } + /// `build_sequence = 2`: a real build's `ManifestRef` is unique; the table's ref uses 1. + const ManifestRef ref{.writer_epoch = victim.writerEpoch(), .build_sequence = 2, .manifest_ordinal = 1}; + writeManifestRaw(*victim.poolBackendPtr(), victim.layout(), ns, ref, {}); + addPrecommitTransition(*victim.poolBackendPtr(), victim.layout(), ns, UInt128(1), "precommit_0", std::nullopt, ref); + }); - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + const auto report = run("admin"); EXPECT_EQ(report.namespaces_removed, 1u); EXPECT_EQ(report.committed_refs_removed, 1u); @@ -816,16 +784,9 @@ TEST(CASDecommission, CountsRealisticEpochPrecommit) EXPECT_EQ(report.edge_deltas_emitted, 2u); } -/// Task 2 review finding 2: the `member_decommission` begin/namespace_removed/end events -/// (CasDecommission.cpp) had no assertion at all. Wire a capturing sink (the `gtest_cas_event_log.cpp` -/// idiom) into `decommissionPoolMember` and check the emitted sequence and its per-namespace detail. -TEST(CASDecommission, EmitsMemberDecommissionEvents) +TEST_F(CASDecommission, EmitsMemberDecommissionEvents) { - auto backend = std::make_shared(); - { - auto victim = openVictim(backend); - makeTableWithRefs(*victim, "victim/db/t1", /*committed=*/1, /*precommits=*/0); - } + auto backend = startVictim(std::make_shared(), oneTable); std::vector seen; (void)decommissionPoolMember(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim", @@ -844,762 +805,2237 @@ TEST(CASDecommission, EmitsMemberDecommissionEvents) EXPECT_EQ(member_events[1].detail.at("precommits"), "0"); EXPECT_EQ(member_events[2].outcome, "end"); EXPECT_EQ(member_events[2].detail.at("namespaces_removed"), "1"); + EXPECT_EQ(member_events[2].detail.at("member_retired"), "1"); + EXPECT_EQ(member_events[2].detail.at("owner_life"), "2"); } -/// Task 3: the manifest-debris / staging / roots drain phases fill their three `DecommissionReport` -/// counters and leave nothing of the victim behind under `staging/` or `roots/`. -TEST(CASDecommission, DrainsDebrisStagingAndRoots) +/// The staging and mountpoint drains empty their prefixes even when the manifest-debris drain retains. +TEST_F(CASDecommission, DrainsDebrisStagingAndRoots) { - auto backend = std::make_shared(); - { - auto victim = openVictim(backend); - makeTableWithRefs(*victim, "victim/db/t1", 1, 0); - seedOrphanManifestBody(*victim, "victim/db/t1"); - } - /// Foreign staging + mountpoint objects, written raw (no writer machinery needed): the victim's - /// writers are fenced by the claim before decommission ever gets here, so these are ordinary debris, - /// not a live in-flight write. - { - OperationForTest seed_op(*backend); - (*seed_op).create("p/staging/victim/upload1.tmp", "x", Retry::once()); - (*seed_op).create("p/staging/victim/upload2.tmp", "x", Retry::once()); - (*seed_op).create("p/roots/victim/clickhouse_access_check_abc", "x", Retry::once()); - } + auto backend = startVictim(std::make_shared(), [](Pool & victim) { oneTableWithDebris(victim); }); + seed("p/staging/victim/upload1.tmp"); + seed("p/staging/victim/upload2.tmp"); + seed("p/roots/victim/clickhouse_access_check_abc"); - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + const auto report = run("admin"); - /// The staging and mountpoint phases are unchanged and still drain completely. The manifest-debris - /// phase retains under the §6 premise (see the note on the helpers above) and reports why, which is - /// what keeps the slot; `RetainsDebrisWhoseEpochSealIsUnconsumed` is that outcome's own test. + /// The manifest-debris retention is `RetainsDebrisWhoseEpochSealIsUnconsumed`'s subject. EXPECT_EQ(report.manifest_debris_removed, 0u); EXPECT_EQ(report.staging_objects_removed, 2u); EXPECT_EQ(report.mountpoint_objects_removed, 1u); - EXPECT_FALSE(report.warnings.empty()) - << "the retained debris is reported, so the incomplete drain is visible"; - - /// Nothing of the victim remains under staging/ or roots/ (scoped LISTs are empty). Those two phases - /// run to completion even though the debris phase retained -- the drain is per-phase, not all-or-nothing. + EXPECT_FALSE(report.warnings.empty()); OperationForTest raw_op(*backend); EXPECT_TRUE((*raw_op).list("p/staging/victim/", "", 10, Retry::once()).keys.empty()); EXPECT_TRUE((*raw_op).list("p/roots/victim/", "", 10, Retry::once()).keys.empty()); } -/// The §6 deletion premise applies to the decommission drain too, and this pins what that COSTS. With no -/// sealed fold cursor for the victim's namespace — the state of a pool whose GC has not folded past the -/// victim's closed epoch — the drain cannot show the debris is unreferenced, so it RETAINS it, says why -/// in `warnings`, and therefore keeps the slot for a later re-run. Delay, not damage: the objects are -/// untouched and a re-run after GC catches up drains them (`DrainsDebrisStagingAndRoots`). -/// -/// This is the visible edge of a real Stage-A limitation, not a test-only artifact: debris under a -/// namespace GC never folds — the pure pre-precommit orphan, whose whole point is that no ref record was -/// ever appended for it — has no cursor to consume any seal, so the premise retains it indefinitely. -/// Reclaiming it needs the sweep's own rework (registers R2/R3, Stage B), which is why the premise ships -/// as the safety floor and not as the reclaim policy. -TEST(CASDecommission, RetainsDebrisWhoseEpochSealIsUnconsumed) +/// With no sealed fold cursor past the victim's epoch the drain cannot show the debris is unreferenced: it +/// retains it, names it in a warning, and leaves the unverified tombstone for a rerun after GC catches up. +TEST_F(CASDecommission, RetainsDebrisWhoseEpochSealIsUnconsumed) { - auto backend = std::make_shared(); String debris_key; - { - auto victim = openVictim(backend); - makeTableWithRefs(*victim, "victim/db/t1", 1, 0); - const ManifestId debris_id = seedOrphanManifestBody(*victim, "victim/db/t1"); - debris_key = victim->layout().manifestKey(debris_id); - /// Deliberately NO `seedFoldedPastVictimEpoch` here — that absence is the subject. - } + startVictim(std::make_shared(), [&](Pool & victim) { debris_key = oneTableWithDebris(victim); }); - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + const auto report = run("admin"); EXPECT_EQ(report.manifest_debris_removed, 0u); - OperationForTest raw_op(*backend); - EXPECT_TRUE((*raw_op).head(debris_key, Retry::once()).has_value()) - << "the body is retained untouched, not deleted and not corrupted"; - ASSERT_FALSE(report.warnings.empty()) - << "a retained manifest is a visible decision -- the operator must be able to see why the drain " - "did not complete"; - bool named = false; - for (const String & w : report.warnings) - if (w.find(debris_key) != String::npos && w.find("seal") != String::npos) - named = true; - EXPECT_TRUE(named) << "the warning names the object and the premise that retained it"; - EXPECT_FALSE(report.slot_removed) - << "an incomplete drain keeps the slot as the resume anchor, exactly as a per-key failure does"; -} - -/// Task 3 fail-close nuance (spec §core "Fail-close"): a per-object failure in the staging/roots drain -/// -- a thrown exception (a transient hiccup) or a `TokenMismatch` outcome (a "listed but raced" miss) -/// -- must record a warning and let the rest of the sweep proceed, never abort the whole phase or the -/// whole command. One staging object throws, the roots object comes back `TokenMismatch`; the OTHER -/// staging object must still be deleted and counted. -TEST(CASDecommission, PerObjectFailureWarnsAndContinuesDrain) + EXPECT_TRUE(exists(debris_key)); + ASSERT_EQ(report.warnings.size(), 1u) << fmt::format("{}", fmt::join(report.warnings, "; ")); + EXPECT_NE(report.warnings.front().find(debris_key), String::npos) << report.warnings.front(); + EXPECT_FALSE(report.member_retired); + expectOwner(2, true, false); +} + +/// A per-object failure in the staging or roots drain, a throw or a `Mismatch`, is a warning and the rest of +/// the drain proceeds. +TEST_F(CASDecommission, PerObjectFailureWarnsAndContinuesDrain) { - auto backend = std::make_shared(); - { - auto victim = openVictim(backend); - makeTableWithRefs(*victim, "victim/db/t1", 1, 0); - } - { - OperationForTest seed_op(*backend); - (*seed_op).create("p/staging/victim/upload_ok.tmp", "x", Retry::once()); - (*seed_op).create("p/staging/victim/upload_throws.tmp", "x", Retry::once()); - (*seed_op).create("p/roots/victim/clickhouse_access_check_abc", "x", Retry::once()); - } + auto backend = startVictim(std::make_shared(), oneTable); + seed("p/staging/victim/upload_ok.tmp"); + seed("p/staging/victim/upload_throws.tmp"); + seed("p/roots/victim/clickhouse_access_check_abc"); backend->failWithThrow("p/staging/victim/upload_throws.tmp"); backend->latch(); backend->failWithTokenMismatch("p/roots/victim/clickhouse_access_check_abc"); - /// The engine reissues an unresolved delete until its own retry window closes, measured on this - /// clock, so the latched fault reaches a genuine give-up with no real time passing. - /// Heap-owned, not a plain stack local: `decommissionPoolMember` installs this clock into the - /// Pool's `boot_ms_fn` background mount-lease renewer, which can still be running on a detached - /// thread after this function returns, so a by-reference capture of a local would dangle. Wrapped - /// (rather than passing `clock->nowFn()`/`clock->sleepFn()` directly) so the closures stored in - /// `PoolConfig` hold the shared_ptr itself, not just the raw `FakeClock*` those methods capture. - auto clock = std::make_shared(); - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim", - /*sink=*/{}, /*request_gc_round=*/{}, - [clock] - { - return clock->nowFn()(); - }, - [clock](uint64_t ms) - { - clock->sleepFn()(ms); - }); + /// The latched fault reaches the engine's give-up on the fake clock, with no real time passing. + auto clock = std::make_shared(); + const auto report = runOnClock("admin", clock); - EXPECT_EQ(report.staging_objects_removed, 1u) - << "the OTHER staging object must still be deleted despite the injected failure on its sibling"; + EXPECT_EQ(report.staging_objects_removed, 1u); EXPECT_EQ(report.mountpoint_objects_removed, 0u); - EXPECT_EQ(report.warnings.size(), 2u) - << "one warning for the thrown exception, one for the TokenMismatch outcome"; - - OperationForTest raw_op(*backend); - EXPECT_FALSE((*raw_op).head("p/staging/victim/upload_ok.tmp", Retry::once()).has_value()) - << "the healthy staging object was actually deleted, not merely skipped"; - EXPECT_TRUE((*raw_op).head("p/staging/victim/upload_throws.tmp", Retry::once()).has_value()) - << "the failing object is left behind (untouched) so a re-run can retry it"; - EXPECT_TRUE((*raw_op).head("p/roots/victim/clickhouse_access_check_abc", Retry::once()).has_value()) - << "TokenMismatch means nothing was actually deleted -- the object survives"; + EXPECT_EQ(report.warnings.size(), 2u); + EXPECT_FALSE(exists("p/staging/victim/upload_ok.tmp")); + EXPECT_TRUE(exists("p/staging/victim/upload_throws.tmp")); + EXPECT_TRUE(exists("p/roots/victim/clickhouse_access_check_abc")); } -/// Opaque physical debris carries no logical owner and therefore cannot widen or redirect -/// decommission's catalog-derived victim set. Task 5's ownership-tree janitor owns that debris. -TEST(CASDecommission, LifelessPhysicalKeyCannotRedirectCatalogOwnedDecommission) +/// Opaque physical debris carries no logical owner, so it cannot widen or redirect the catalog-derived victim +/// set; the namespace janitor owns it. +TEST_F(CASDecommission, LifelessPhysicalKeyCannotRedirectCatalogOwnedDecommission) { - auto backend = std::make_shared(); - String lifeless; - { - auto victim = openVictim(backend); - makeTableWithRefs(*victim, "victim/db/t1", 1, 0); - /// Hand-built: no helper can mint the un-incarnated shape any more. - lifeless = victim->layout().casRefsPrefix() + String("victim/db/t1/_log/") - + renderRefTxnId(RefTxnId{1, 1}) + ".zst"; - OperationForTest seed_op(*backend); - ASSERT_TRUE(std::holds_alternative((*seed_op).create(lifeless, "garbage", Retry::once()))); - } + startVictim(std::make_shared(), oneTable); + /// Hand-built: no helper can mint the un-incarnated shape any more. + const String lifeless = layout.casRefsPrefix() + String("victim/db/t1/_log/") + renderRefTxnId(RefTxnId{1, 1}) + ".zst"; + seed(lifeless); - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + const auto report = run("admin"); EXPECT_EQ(report.namespaces_removed, 1u); - OperationForTest raw_op(*backend); - EXPECT_TRUE((*raw_op).head(lifeless, Retry::once()).has_value()) - << "decommission must neither adopt nor delete an unowned physical life key"; + EXPECT_TRUE(exists(lifeless)); } -/// The manifest-debris drain honors the same tolerate-and-continue contract as -/// `deleteListedPrefix`: a per-key `deleteExact` failure becomes a warning, while the namespace -/// erasure and subsequent staging drain continue. Protection reads now use opaque physical life -/// prefixes, so a logical-name substring can no longer target an otherwise unlisted namespace. -TEST(CASDecommission, ManifestDebrisDeleteFailureWarnsAndContinues) +/// A per-key failure in the manifest-debris drain is a warning; the namespace drop and the staging drain go on. +TEST_F(CASDecommission, ManifestDebrisDeleteFailureWarnsAndContinues) { - auto backend = std::make_shared(); String debris_key; - { - auto victim = openVictim(backend); - makeTableWithRefs(*victim, "victim/db/t1", 1, 0); - const ManifestId debris_id = seedOrphanManifestBody(*victim, "victim/db/t1"); - debris_key = victim->layout().manifestKey(debris_id); - } + auto backend = startVictim(std::make_shared(), [&](Pool & victim) { debris_key = oneTableWithDebris(victim); }); backend->failWithThrow(debris_key); backend->latch(); - { - OperationForTest seed_op(*backend); - (*seed_op).create("p/staging/victim/upload_ok.tmp", "x", Retry::once()); - } - - /// The engine reissues an unresolved delete until its own retry window closes, measured on this - /// clock, so the latched fault reaches a genuine give-up with no real time passing. - /// Heap-owned, not a plain stack local: `decommissionPoolMember` installs this clock into the - /// Pool's `boot_ms_fn` background mount-lease renewer, which can still be running on a detached - /// thread after this function returns, so a by-reference capture of a local would dangle. Wrapped - /// (rather than passing `clock->nowFn()`/`clock->sleepFn()` directly) so the closures stored in - /// `PoolConfig` hold the shared_ptr itself, not just the raw `FakeClock*` those methods capture. - auto clock = std::make_shared(); - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim", - /*sink=*/{}, /*request_gc_round=*/{}, - [clock] - { - return clock->nowFn()(); - }, - [clock](uint64_t ms) - { - clock->sleepFn()(ms); - }); + seed("p/staging/victim/upload_ok.tmp"); - EXPECT_EQ(report.namespaces_removed, 1u) - << "victim/db/t1's namespace erasure (Task 2) is untouched by either injected failure"; - EXPECT_EQ(report.manifest_debris_removed, 0u); - EXPECT_EQ(report.warnings.size(), 1u) - << "the thrown per-key delete must keep the retirement tail fail-closed"; - EXPECT_EQ(report.staging_objects_removed, 1u) - << "the staging phase still ran to completion after the manifest-debris phase's failures -- " - "the whole command did not abort"; + /// The latched fault reaches the engine's give-up on the fake clock, with no real time passing. + auto clock = std::make_shared(); + const auto report = runOnClock("admin", clock); - OperationForTest raw_op(*backend); - EXPECT_TRUE((*raw_op).head(debris_key, Retry::once()).has_value()) - << "the failing object is left behind (untouched) so a re-run can retry it"; + EXPECT_EQ(report.namespaces_removed, 1u); + EXPECT_EQ(report.manifest_debris_removed, 0u); + EXPECT_EQ(report.warnings.size(), 1u); + EXPECT_EQ(report.staging_objects_removed, 1u); + EXPECT_TRUE(exists(debris_key)); } -/// GC owns the completed catalog-row deletion. Once it drains the row, a clean decommission retry -/// removes the mutable slot objects and tombstones the owner anchor. -TEST(CASDecommission, RemovesMutableSlotAndRefusesTombstonedRerun) +/// A rerun over a verified retirement writes a fresh tombstone and mark of its own each time; the control +/// objects stay. +TEST_F(CASDecommission, RerunOverAVerifiedRetirementRetiresAgain) { - auto backend = std::make_shared(); - { - auto victim = openVictim(backend); - makeTableWithRefs(*victim, "victim/db/t1", 1, 0); - } - const auto pending = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); - EXPECT_FALSE(pending.slot_removed); - EXPECT_FALSE(pending.warnings.empty()); + auto backend = startVictim(std::make_shared(), oneTable); + const auto first = run("admin"); + EXPECT_TRUE(first.member_retired); + EXPECT_EQ(first.namespaces_pending_gc, 1u); drainCompletedNamespaceRemovals(backend); - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin2"}, "victim"); - EXPECT_TRUE(report.slot_removed); - EXPECT_TRUE(report.warnings.empty()); - OperationForTest raw_op(*backend); - EXPECT_FALSE((*raw_op).head("p/gc/server-roots/victim/mount", Retry::once()).has_value()); - const auto owner = (*raw_op).read("p/gc/server-roots/victim/owner", Retry::once()); - ASSERT_TRUE(owner.has_value()); - EXPECT_TRUE(decodeOwner(owner->bytes).retired_at_ms.has_value()); - EXPECT_FALSE((*raw_op).head("p/gc/server-roots/victim/epoch", Retry::once()).has_value()); - expectThrowsCode(ErrorCodes::CORRUPTED_DATA, [&] - { - decommissionPoolMember(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "a2"}, "victim"); - }); + const auto second = run("admin2"); + EXPECT_TRUE(second.member_retired); + EXPECT_TRUE(second.warnings.empty()); + EXPECT_EQ(second.namespaces_pending_gc, 0u); + EXPECT_TRUE(exists(layout.mountKey("victim"))); + EXPECT_TRUE(exists(layout.epochKey("victim"))); + expectOwner(5, true, true); + + const auto third = run("a2"); + EXPECT_TRUE(third.member_retired); + EXPECT_TRUE(third.warnings.empty()); + expectOwner(7, true, true); } -/// Triage #9: a successor may reclaim the same UUID immediately after the decommission admin writes -/// its farewell. The retirement tail must use the farewell/claimed-epoch tokens captured around that -/// release, delete `mount` first, and stop on its `TokenMismatch`; re-reading current tokens would -/// delete the live successor's control objects and falsely report the slot removed. -TEST(CASDecommission, SuccessorReclaimFencesSlotRetirementTail) +/// A resolve read settles an ambiguous tombstone write: the bytes already match, so the tombstone is committed. +TEST_F(CASDecommission, OwnerTombstoneAmbiguousSuccessResolvesToCommitted) { - auto backend = std::make_shared(); - { auto victim = openVictim(backend); } - backend->armForSuccessorReclaim(); - - std::vector seen; - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim", - [&](const CasEvent & event) { seen.push_back(event); }); - - ASSERT_TRUE(backend->successorInjected()); - EXPECT_FALSE(report.slot_removed); - ASSERT_EQ(report.warnings.size(), 1u); - EXPECT_NE(report.warnings.front().find("p/gc/server-roots/victim/mount"), String::npos); - EXPECT_NE(report.warnings.front().find("mismatch"), String::npos); - - OperationForTest raw_op(*backend); - const auto mount = (*raw_op).read("p/gc/server-roots/victim/mount", Retry::once()); - ASSERT_TRUE(mount.has_value()); - EXPECT_EQ(PersistedEtag::capture(mount->etag).value, backend->successorMountValue()); - EXPECT_EQ(mount->bytes, backend->successorMountBytes()); + auto backend = startVictim(); + backend->injectAmbiguousLandedWrite(layout.ownerKey("victim")); - const auto epoch = (*raw_op).read("p/gc/server-roots/victim/epoch", Retry::once()); - ASSERT_TRUE(epoch.has_value()); - EXPECT_EQ(PersistedEtag::capture(epoch->etag).value, backend->successorEpochValue()); - EXPECT_EQ(epoch->bytes, backend->successorEpochBytes()); - EXPECT_TRUE((*raw_op).read("p/gc/server-roots/victim/owner", Retry::once()).has_value()); + const auto report = run("admin"); - ASSERT_FALSE(seen.empty()); - EXPECT_EQ(seen.back().outcome, "end"); - EXPECT_EQ(seen.back().detail.at("slot_removed"), "0"); + EXPECT_TRUE(report.member_retired); + EXPECT_TRUE(report.warnings.empty()); + expectOwner(3, true, true); } -/// A successor can also restart after both stale mutable objects were deleted but before `owner` is -/// retired. Mere presence of either freshly recreated mutable object must stop owner retirement. -TEST(CASDecommission, SuccessorReclaimAfterEpochDeleteKeepsOwnerAnchor) +/// Delegates every op to `inner`, except `remove`: while armed, a key under `fail_prefix` throws a transient +/// failure the engine reissues until it gives up. `disarm()` clears it. +class FailDeletesUnderPrefixBackend : public Backend { - auto backend = std::make_shared(); - { auto victim = openVictim(backend); } - - const String owner_key = "p/gc/server-roots/victim/owner"; - OperationForTest raw_op(*backend); - const auto original_owner = (*raw_op).read(owner_key, Retry::once()); - ASSERT_TRUE(original_owner.has_value()); - backend->armForSuccessorReclaim(); - - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); +public: + FailDeletesUnderPrefixBackend(std::shared_ptr inner_, String fail_prefix_) + : inner(std::move(inner_)), fail_prefix(std::move(fail_prefix_)) + { + } - ASSERT_TRUE(backend->successorInjected()); - EXPECT_FALSE(report.slot_removed); - EXPECT_FALSE(report.warnings.empty()); - EXPECT_EQ(backend->ownerRewriteAttempts(), 0u); + void disarm() { armed = false; } - const auto owner = (*raw_op).read(owner_key, Retry::once()); - ASSERT_TRUE(owner.has_value()); - EXPECT_EQ(owner->etag, original_owner->etag); - EXPECT_EQ(owner->bytes, original_owner->bytes); + bool supportsListTokens() const override { return inner->supportsListTokens(); } - const auto mount = (*raw_op).read("p/gc/server-roots/victim/mount", Retry::once()); - ASSERT_TRUE(mount.has_value()); - EXPECT_EQ(PersistedEtag::capture(mount->etag).value, backend->successorMountValue()); - EXPECT_EQ(mount->bytes, backend->successorMountBytes()); + std::optional read(const String & key, TransportAccess & access) override { return inner->read(key, access); } + std::optional head(const String & key, TransportAccess & access) override { return inner->head(key, access); } + RawListPage list(const String & prefix, const String & cursor, size_t limit, TransportAccess & access) override { return inner->list(prefix, cursor, limit, access); } + RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override + { + if (armed && key.starts_with(fail_prefix)) + throw Poco::TimeoutException("injected transient delete failure for " + key); + return inner->remove(key, expected_value, access); + } + void removeManyWriteOnce(const std::vector & keys, TransportAccess & access) override { inner->removeManyWriteOnce(keys, access); } + size_t bulkDeleteKeyLimit() const override { return inner->bulkDeleteKeyLimit(); } + std::expected write(const String & key, const String & bytes, + const std::optional & expected_value, TransportAccess & access) override + { + return inner->write(key, bytes, expected_value, access); + } + std::unique_ptr stream(const String & key, TransportAccess & access) override { return inner->stream(key, access); } + void publish(const BlobPublishRequest & request, TransportAccess & access) override { inner->publish(request, access); } + DB::Cas::Dialect dialect() const override { return inner->dialect(); } - const auto epoch = (*raw_op).read("p/gc/server-roots/victim/epoch", Retry::once()); - ASSERT_TRUE(epoch.has_value()); - EXPECT_EQ(PersistedEtag::capture(epoch->etag).value, backend->successorEpochValue()); - EXPECT_EQ(epoch->bytes, backend->successorEpochBytes()); -} +private: + std::shared_ptr inner; + String fail_prefix; + bool armed = true; +}; -/// Triage #9 control: absent a successor interleaving, the fenced tail removes both mutable control -/// objects, tombstones the owner anchor, and preserves the existing successful `slot_removed=1` result. -TEST(CASDecommission, FencedSlotRetirementTailRetiresUncontendedSlot) +/// A drain failure under the roots prefix leaves this run's unverified tombstone. Once the fault is +/// cleared, a rerun resumes over it: the already-erased namespace is counted as +/// `namespaces_already_removed`, the leftover roots object is swept and the retirement is verified. +TEST_F(CASDecommission, FailedDrainLeavesAnUnverifiedTombstoneThenResumes) { - auto backend = std::make_shared(); - { auto victim = openVictim(backend); } + auto inner = startVictim(std::make_shared(), oneTable); + seed("p/roots/victim/loose_file"); + auto failing = std::make_shared(inner, "p/roots/victim/"); + victim_backend = failing; - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + /// The admin's boot clock is the run's clock, so its lease must outlast the retries' fast-forward. + auto clock = std::make_shared(); + const auto first = runOnClock( + "a1", clock, PoolConfig{.pool_prefix = "p", .server_root_id = "", .mount_lease_ttl_ms = std::chrono::milliseconds(300'000)}); + EXPECT_FALSE(first.warnings.empty()); + EXPECT_FALSE(first.member_retired); + expectOwner(2, true, false); + EXPECT_TRUE(exists(layout.mountKey("victim"))); + + failing->disarm(); + const auto second = run("a2"); + EXPECT_TRUE(second.warnings.empty()); + EXPECT_TRUE(second.member_retired); + EXPECT_EQ(second.mountpoint_objects_removed, 1u); + EXPECT_EQ(second.namespaces_already_removed, 1u); + EXPECT_EQ(second.namespaces_pending_gc, 1u); + + drainCompletedNamespaceRemovals(inner); + const auto third = run("a3"); + EXPECT_TRUE(third.warnings.empty()); + EXPECT_TRUE(third.member_retired); + EXPECT_EQ(third.namespaces_pending_gc, 0u); +} + +/// A per-key throw inside the manifest-debris drain leaves the unverified tombstone too. The debris stays +/// retained once the fault is cleared (no sealed fold cursor), so the rerun is not verified either. +TEST_F(CASDecommission, ManifestDebrisFailureLeavesAnUnverifiedTombstone) +{ + String debris_key; + auto backend = startVictim(std::make_shared(), [&](Pool & victim) { debris_key = oneTableWithDebris(victim); }); + backend->failWithThrow(debris_key); + backend->latch(); + + auto clock = std::make_shared(); + const auto first = runOnClock("a1", clock); + EXPECT_FALSE(first.warnings.empty()); + EXPECT_FALSE(first.member_retired); + EXPECT_EQ(first.manifest_debris_removed, 0u); + expectOwner(2, true, false); + EXPECT_TRUE(exists(debris_key)); + + backend->disarm(); + const auto second = run("a2"); + EXPECT_EQ(second.namespaces_already_removed, 1u); + EXPECT_EQ(second.manifest_debris_removed, 0u); + EXPECT_FALSE(second.member_retired); + expectOwner(3, true, false); + EXPECT_TRUE(exists(debris_key)); + EXPECT_TRUE(exists(layout.mountKey("victim"))); +} + +/// The farewell deadline is bound to the admin's boot clock, which the run's fake clock must drive too. A fake +/// clock far beyond any real host's boot time makes a split between the two clocks fail on every machine. +TEST_F(CASDecommission, DrainClockUnifiesWithTheFarewellBootClockRegardlessOfHostUptime) +{ + startVictim(); + auto clock = farClock(); + const auto report = runOnClock("a1", clock); + + EXPECT_TRUE(report.warnings.empty()); + EXPECT_TRUE(report.member_retired); + EXPECT_TRUE(slot().isCleanRelease()); +} + +/// The planes `Pool::openForDecommission` builds retry during opening, before the run installs its clock on +/// the pool; they must pace on the fake sleep from construction, or a retry deadline on the frozen clock +/// never elapses. +TEST_F(CASDecommission, OpeningRetriesPaceOnTheSameFakeClockAndSleepAsTheDrain) +{ + auto backend = startVictim(std::make_shared()); + backend->failReadNTimes(layout.ownerKey("victim"), /*times=*/5); + + auto clock = farClock(); + + const auto started = std::chrono::steady_clock::now(); + const auto report = runOnClock("a1", clock); + const auto wall_elapsed = std::chrono::steady_clock::now() - started; + + EXPECT_TRUE(report.warnings.empty()); + EXPECT_TRUE(report.member_retired); + EXPECT_TRUE(slot().isCleanRelease()); + EXPECT_FALSE(clock->sleeps.empty()); + EXPECT_LT(wall_elapsed, std::chrono::seconds(5)); +} + +/// The victim restarts during its own decommission: its owner check passes before the tombstone, its mount +/// claim waits on the admin's live lease, the admin tombstones and farewells during that wait, the victim +/// reclaims the farewell and its recheck sees the tombstone. +TEST_F(CASDecommission, VictimClaimingAfterTheTombstoneFencesItselfAndFails) +{ + auto backend = startVictim(); + const UInt128 victim_uuid = owner().server_uuid; + auto admin = Pool::openForDecommission(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + const uint64_t admin_epoch = admin->writerEpoch(); + + auto fake_boot = std::make_shared>(0); + bool decommissioned = false; + PoolConfig victim_config{ + .pool_prefix = "p", .server_root_id = "victim", + .boot_ms_fn = [fake_boot] { return fake_boot->load(); }, + .wait_sleep_fn = [&, fake_boot](uint64_t ms) + { + *fake_boot += ms; + if (std::exchange(decommissioned, true)) + return; + tombstoneOwner(); + admin.reset(); /// the clean farewell the victim's next poll reclaims + }}; + expectThrowsCodeWithMessage(ErrorCodes::CORRUPTED_DATA, "decommissioned", [&] { Pool::open(backend, victim_config); }); + ASSERT_TRUE(decommissioned); + const MountLease body = slot(); + EXPECT_EQ(body.server_uuid, victim_uuid); + EXPECT_GT(body.writer_epoch, admin_epoch); + EXPECT_TRUE(body.gc_fenced); + EXPECT_TRUE(owner().retired_at_ms.has_value()); +} + + +namespace +{ +/// While armed, `_pool_meta` cannot be probed: an admin self-remount stops at its identity probe and retries +/// on a backoff the teardown interrupts. +class PoolMetaUnreadableBackend : public InMemoryBackend +{ +public: + std::atomic armed = false; + + std::optional head(const String & key, TransportAccess & access) override + { + if (armed && key == Layout("p").poolMetaKey()) + throw Poco::TimeoutException("pool meta fixture: unreadable"); + return InMemoryBackend::head(key, access); + } +}; +} + +/// The tombstone's first attempt goes unanswered and the admin's lease deadline passes before the retry. +/// The retry is refused locally, so a victim restarted after the deadline runs under the life it reads. +TEST_F(CASDecommission, TombstoneRetryAfterTheLeaseDeadlineIsNotSent) +{ + auto backend = startVictim(std::make_shared()); + auto clock = farClock(); + bool stalled = false; + backend->onBeforeWrite(layout.ownerKey("victim"), [&] + { + if (std::exchange(stalled, true)) + return; + /// The lapsed lease sends the admin into a self-remount; keep it from racing the teardown. + backend->armed = true; + clock->now += 31'000; /// past the default 30 s TTL + backend->injectAmbiguousWrite(layout.ownerKey("victim")); + }); + + EXPECT_THROW(runOnClock("a1", clock), DB::Exception); + + ASSERT_TRUE(stalled); + backend->armed = false; + expectOwner(1, false, false); + auto restarted = Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "victim", .unsafe_remount_no_delay = true}); + EXPECT_TRUE(restarted->mayMutate()); + expectOwner(1, false, false); +} + +/// A local fault after the tombstone went out leaves its fate unknown, and the read-back finds the old owner: +/// the store may still apply the tombstone. The admin leaves its live body instead of a clean farewell, so a +/// restarted victim waits for a fence before it runs. +TEST_F(CASDecommission, UnresolvedTombstoneLeavesNoFarewell) +{ + auto backend = startVictim(); + const UInt128 victim_uuid = owner().server_uuid; + bool sent = false; + backend->onBeforeWrite(layout.ownerKey("victim"), [&] + { + if (!std::exchange(sent, true)) + throw std::runtime_error("fixture: a local fault after the tombstone was sent"); + }); + + EXPECT_THROW(runOnClock("a1", farClock()), std::runtime_error); + + ASSERT_TRUE(sent); + expectOwner(1, false, false); + const MountLease left = slot(); + EXPECT_EQ(left.server_uuid, victim_uuid); + EXPECT_FALSE(left.isCleanRelease()); + EXPECT_FALSE(left.gc_fenced); + + auto fake_boot = std::make_shared>(0); + bool fenced = false; + PoolConfig victim_config{ + .pool_prefix = "p", .server_root_id = "victim", + .boot_ms_fn = [fake_boot] { return fake_boot->load(); }, + .wait_sleep_fn = [&, fake_boot](uint64_t ms) + { + *fake_boot += ms; + if (!std::exchange(fenced, true)) + fenceSlot(); + }}; + auto restarted = Pool::open(backend, victim_config); + EXPECT_TRUE(fenced) << "the victim reclaimed the slot without waiting"; + EXPECT_TRUE(restarted->mayMutate()); + EXPECT_GT(restarted->writerEpoch(), left.writer_epoch); +} + +/// The owner is rewritten between the claim and the tombstone: the tombstone's precondition fails, the run +/// aborts, the rewrite stays and nothing is drained. +TEST_F(CASDecommission, OwnerRewrittenBeforeTheTombstoneAbortsTheRun) +{ + auto backend = startVictim(); + seed("p/roots/victim/loose_file"); + String rewritten; + backend->onBeforeWrite(layout.ownerKey("victim"), [&] + { + if (!rewritten.empty()) + return; + OperationForTest op(*backend); + const auto current = (*op).read(layout.ownerKey("victim"), Retry::once()); + OwnerObject owner = decodeOwner(current->bytes); + owner.owner_life += 1; + rewritten = encodeOwner(owner); + ASSERT_TRUE(std::holds_alternative( + (*op).replace(layout.ownerKey("victim"), rewritten, current->etag, Retry::once()))); + }); + + expectThrowsCode(ErrorCodes::ABORTED, [&] { run("a1"); }); + + ASSERT_FALSE(rewritten.empty()); + expectBytes(*backend, layout.ownerKey("victim"), rewritten); + EXPECT_TRUE(exists("p/roots/victim/loose_file")); +} + +/// A member last written by 26.6.4: owner without a life, counter without a nonce, its clean farewell. A +/// decommission that loses its claim rolls back to life 2, and the member restarts under it. +TEST_F(CASDecommission, A2664MemberRollsBackAndRestarts) +{ + auto backend = startVictim(); + const String hex = u128ToHex(owner().server_uuid); + { + OperationForTest op(*backend); + const auto put = [&](const String & key, const String & bytes) + { + const auto current = (*op).read(key, Retry::once()); + ASSERT_TRUE(current.has_value()); + ASSERT_TRUE(std::holds_alternative((*op).replace(key, bytes, current->etag, Retry::once()))); + }; + put(layout.ownerKey("victim"), "{\"type\":\"cas_owner\",\"v\":1}\n{\"server_uuid\":\"" + hex + "\"}\n"); + put(layout.epochKey("victim"), "{\"type\":\"cas_epoch\",\"v\":1}\n{\"next_writer_epoch\":\"5\"}\n"); + put(layout.mountKey("victim"), "{\"type\":\"cas_mount_lease\",\"v\":1}\n{\"server_uuid\":\"" + hex + + "\",\"writer_epoch\":\"4\",\"hostname\":\"h\",\"pid\":1,\"started_at_ms\":1,\"seq\":\"3\",\"expires_at_ms\":1," + "\"min_active_build_sequence\":\"18446744073709551615\",\"gc_fenced\":false," + "\"write_attempt_id\":\"0000000000000000000000000000000a\"}\n"); + } + bool fenced = false; + backend->onWriteCommitted(layout.ownerKey("victim"), [&] + { + if (!std::exchange(fenced, true)) + fenceSlot(); + }); + + const auto report = run("a1"); + + ASSERT_TRUE(fenced); + EXPECT_FALSE(report.member_retired); + expectOwner(2, false, false); /// 26.6.4 life 0, tombstone 1, rollback 2 + auto restarted = Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "victim", .unsafe_remount_no_delay = true}); + EXPECT_GT(restarted->writerEpoch(), 5u); + expectOwner(2, false, false); +} + +TEST_F(CASDecommission, KeepsEpochAndFarewellMount) +{ + uint64_t victim_epoch = 0; + auto backend = startVictim(std::make_shared(), [&](Pool & victim) { victim_epoch = victim.writerEpoch(); }); + const UInt128 victim_uuid = owner().server_uuid; + + const auto report = run("admin"); + + EXPECT_TRUE(report.warnings.empty()); + EXPECT_TRUE(report.member_retired); + const MountLease body = slot(); + EXPECT_EQ(body.writer_epoch, victim_epoch + 1) << "the admin's own clean farewell stays in the slot"; + EXPECT_TRUE(body.isCleanRelease()); + EXPECT_FALSE(body.gc_fenced); + OperationForTest raw_op(*backend); + const auto epoch = (*raw_op).read(layout.epochKey("victim"), Retry::once()); + ASSERT_TRUE(epoch.has_value()); + EXPECT_EQ(decodeServerEpoch(epoch->bytes).next_writer_epoch, victim_epoch + 2) << "the counter is never deleted"; + expectOwner(3, true, true); /// claim 1, tombstone 2, verified mark 3 + EXPECT_EQ(owner().server_uuid, victim_uuid); +} + +/// The admin is GC-fenced between its claim and its tombstone: the confirm sees the fence and the +/// tombstone is rolled back to a fresh, unretired life. +TEST_F(CASDecommission, FenceBeforeTombstoneUntombstonesAndFails) +{ + auto backend = startVictim(); + seed("p/roots/victim/loose_file"); + bool fired = false; + backend->onBeforeWrite(layout.ownerKey("victim"), [&] + { + if (std::exchange(fired, true)) + return; + fenceSlot(); + }); + + const auto report = run("admin"); + + ASSERT_TRUE(fired); + EXPECT_FALSE(report.member_retired); + EXPECT_EQ(report.warnings.size(), 1u) << fmt::format("{}", fmt::join(report.warnings, "; ")); + expectOwner(3, false, false); /// claim 1, tombstone 2, rollback 3 + EXPECT_TRUE(exists("p/roots/victim/loose_file")) << "nothing was drained"; +} + +/// A renewal between the claim and the confirm changes the ETag and the attempt id, and the confirm must not +/// care; a fence, a farewell, another epoch or another uuid is a loss. +TEST_F(CASDecommission, ConfirmAcceptsARenewalBetweenClaimAndConfirm) +{ + auto backend = std::make_shared(); + OperationForTest op(*backend); + MountLease body{.server_uuid = UInt128(7), .writer_epoch = 5, .hostname = "h", .pid = 1, + .started_at_ms = 1, .seq = 1, .expires_at_ms = 2, .min_active_build_sequence = 0, + .gc_fenced = false, .write_attempt_id = UInt128(11)}; + ASSERT_TRUE(std::holds_alternative((*op).create(layout.mountKey("v"), encodeMountLease(body), Retry::once()))); + EXPECT_TRUE(adminClaimStillHeld(*op, layout, "v", UInt128(7), 5)); + + MountLease renewed = body; + renewed.seq = 2; + renewed.write_attempt_id = UInt128(12); + ASSERT_TRUE(std::holds_alternative((*op).replace(layout.mountKey("v"), encodeMountLease(renewed), + (*op).read(layout.mountKey("v"), Retry::once())->etag, Retry::once()))); + EXPECT_TRUE(adminClaimStillHeld(*op, layout, "v", UInt128(7), 5)); + + EXPECT_FALSE(adminClaimStillHeld(*op, layout, "v", UInt128(7), 6)) << "another epoch"; + EXPECT_FALSE(adminClaimStillHeld(*op, layout, "v", UInt128(8), 5)) << "another uuid"; + ASSERT_TRUE(std::holds_alternative((*op).replace(layout.mountKey("v"), encodeMountLease(renewed.fencedCopy()), + (*op).read(layout.mountKey("v"), Retry::once())->etag, Retry::once()))); + EXPECT_FALSE(adminClaimStillHeld(*op, layout, "v", UInt128(7), 5)) << "fenced"; + MountLease farewell = renewed; + farewell.min_active_build_sequence = std::numeric_limits::max(); + ASSERT_TRUE(std::holds_alternative((*op).replace(layout.mountKey("v"), encodeMountLease(farewell), + (*op).read(layout.mountKey("v"), Retry::once())->etag, Retry::once()))); + EXPECT_FALSE(adminClaimStillHeld(*op, layout, "v", UInt128(7), 5)) << "farewelled"; + EXPECT_FALSE(adminClaimStillHeld(*op, layout, "absent", UInt128(7), 5)); +} + +/// Another server took the server_root_id between the admin's claim and its tombstone (a rejoin). The +/// tombstone stops before it writes anything. +TEST_F(CASDecommission, StopsWhenTheOwnerUuidChanged) +{ + auto backend = startVictim(std::make_shared()); + backend->arm(); + seed("p/roots/victim/loose_file"); + + expectThrowsCode(ErrorCodes::ABORTED, [&] { run("admin"); }); + + ASSERT_TRUE(backend->fired()); + EXPECT_EQ(owner().server_uuid, UInt128(0xBEEF)); + EXPECT_FALSE(owner().retired_at_ms.has_value()) << "the new owner was not tombstoned"; + EXPECT_TRUE(exists("p/roots/victim/loose_file")) << "nothing was drained"; +} + +/// An absent counter is refused, whether or not an owner is left, and never re-created; nothing is written. +TEST_F(CASDecommission, AbsentCounterIsRefused) +{ + for (const bool owner_left : {true, false}) + { + SCOPED_TRACE(owner_left); + startVictim(); + OperationForTest raw_op(*victim_backend); + ASSERT_EQ((*raw_op).removeCurrent(layout.epochKey("victim"), Retry::once()), Removal::Removed); + if (!owner_left) + ASSERT_EQ((*raw_op).removeCurrent(layout.ownerKey("victim"), Retry::once()), Removal::Removed); + const auto before = controlObjects(); + + expectThrowsCodeWithMessage(ErrorCodes::CORRUPTED_DATA, "no epoch counter", [&] { run("admin"); }); + + EXPECT_EQ(controlObjects(), before); + } +} + +/// The owner was deleted by hand while the counter survived: the run recovers the uuid from the lease, +/// re-creates the owner over the empty subtree and retires it. +TEST_F(CASDecommission, OwnerDeletedByHandIsRecreatedFromTheLease) +{ + auto backend = startVictim(); + OperationForTest raw_op(*backend); + ASSERT_EQ((*raw_op).removeCurrent(layout.ownerKey("victim"), Retry::once()), Removal::Removed); + + const auto report = run("a2"); + + EXPECT_TRUE(report.warnings.empty()); + EXPECT_TRUE(report.member_retired); + expectOwner(3, true, true); + EXPECT_TRUE(exists(layout.epochKey("victim"))); +} + +TEST_F(CASDecommission, UnconfirmedFarewellIsNotRetired) +{ + auto backend = startVictim(std::make_shared()); + backend->arm(); + DB::Cas::tests::ScopedLoggerCapture capture("CasDecommission", "warning"); + + const auto report = run("admin"); + + ASSERT_TRUE(backend->fired()); + EXPECT_FALSE(report.member_retired); + ASSERT_FALSE(report.warnings.empty()); + for (const String & warning : report.warnings) + EXPECT_NE(capture.captured().find(warning), String::npos) << "not logged: " << warning; + expectOwner(2, true, false); + EXPECT_TRUE(slot().gc_fenced); +} + +/// The mark is written on the control plane after the admin pool is gone, and does not wait for GC to delete +/// the victim's catalog rows. +TEST_F(CASDecommission, VerifiedMarkDoesNotWaitForCatalogGc) +{ + auto backend = startVictim(std::make_shared(), oneTable); + const auto report = run("admin"); + EXPECT_EQ(report.namespaces_removed, 1u); + EXPECT_EQ(report.namespaces_pending_gc, 1u); + EXPECT_TRUE(report.warnings.empty()); + EXPECT_TRUE(report.member_retired); + expectOwner(3, true, true); +} + +/// A rerun over an earlier retirement, verified or not, loses its claim after its tombstone: nothing is rolled +/// back and the owner stays retired. An unverified tombstone landed while its run held the claim, so no +/// member can be running under it. +TEST_F(CASDecommission, RollbackKeepsAPreviousRetirement) +{ + for (const bool verified : {true, false}) + { + SCOPED_TRACE(verified); + auto backend = startVictim(); + if (verified) + ASSERT_TRUE(run("a1").member_retired); + else + tombstoneOwner(); + const uint64_t life_before = owner().owner_life; + + bool fired = false; + backend->onWriteCommitted(layout.ownerKey("victim"), [&] + { + if (std::exchange(fired, true)) + return; + fenceSlot(); /// the rerun's tombstone landed; its claim is lost now + }); + const auto rerun = run("a2"); + + ASSERT_TRUE(fired); + EXPECT_FALSE(rerun.member_retired); + EXPECT_EQ(rerun.warnings.size(), 1u) << fmt::format("{}", fmt::join(rerun.warnings, "; ")); + expectOwner(life_before + 1, true, verified); /// the rerun's tombstone, kept + } +} + +/// Three runs: one verifies, one refreshes the tombstone and dies, one reruns and is fenced before its +/// confirm. The mark survives all three. +TEST_F(CASDecommission, RefreshKeepsTheVerifiedMark) +{ + auto backend = startVictim(); + ASSERT_TRUE(run("a1").member_retired); + ASSERT_TRUE(owner().retirement_verified); + + tombstoneOwner(); /// a second run's tombstone, then death: life 4, mark kept + ASSERT_TRUE(owner().retirement_verified); + + bool fired = false; + backend->onWriteCommitted(layout.ownerKey("victim"), [&] + { + if (std::exchange(fired, true)) + return; + fenceSlot(); + }); + const auto third = run("a3"); + ASSERT_TRUE(fired); + EXPECT_FALSE(third.member_retired); + expectOwner(5, true, true); +} + +namespace +{ + +/// What an operator leaves after clearing a verified tombstone by hand to restart the member: the mark of the +/// retired life stays. +void clearTombstoneByHand(Backend & backend, const Layout & layout, const String & srid) +{ + OperationForTest op(backend); + const auto current = (*op).read(layout.ownerKey(srid), Retry::once()); + ASSERT_TRUE(current.has_value()); + OwnerObject owner = decodeOwner(current->bytes); + ASSERT_TRUE(owner.retirement_verified); + owner.retired_at_ms.reset(); + ASSERT_TRUE(std::holds_alternative( + (*op).replace(layout.ownerKey(srid), encodeOwner(owner), current->etag, Retry::once()))); +} + +} + +/// The member restarted over a hand-cleared tombstone. The next decommission loses its claim after its +/// tombstone: the mark of the earlier life must not stop the rollback. +TEST_F(CASDecommission, HandClearedTombstoneStillRollsBackALostClaim) +{ + auto backend = startVictim(); + ASSERT_TRUE(run("a1").member_retired); + clearTombstoneByHand(*backend, layout, "victim"); + { auto victim = openVictim(backend); } + + bool fired = false; + backend->onWriteCommitted(layout.ownerKey("victim"), [&] + { + if (std::exchange(fired, true)) + return; + fenceSlot(); + }); + const auto second = run("a2"); + + ASSERT_TRUE(fired); + EXPECT_FALSE(second.member_retired); + EXPECT_EQ(second.warnings.size(), 1u) << fmt::format("{}", fmt::join(second.warnings, "; ")); + expectOwner(5, false, false); /// first run 3, second tombstone 4, rollback 5 +} + +/// The member restarted over a hand-cleared tombstone. The next decommission's farewell is fenced: without a +/// mark of its own life the run reports rerun. +TEST_F(CASDecommission, HandClearedTombstoneIsNotARetirementOfTheNextRun) +{ + auto backend = startVictim(std::make_shared()); + ASSERT_TRUE(run("a1").member_retired); + clearTombstoneByHand(*backend, layout, "victim"); + { auto victim = openVictim(backend); } + backend->arm(); + + const auto second = run("a2"); + + ASSERT_TRUE(backend->fired()); + EXPECT_FALSE(second.member_retired); + expectOwner(4, true, false); +} + +/// A run dies after each step; a rerun completes over the run's own tombstone. +TEST_F(CASDecommission, ResumesOverItsOwnTombstone) +{ + enum class DieAt { AfterTombstone, AfterFarewell, AfterMark }; + for (const DieAt die_at : {DieAt::AfterTombstone, DieAt::AfterFarewell, DieAt::AfterMark}) + { + SCOPED_TRACE(static_cast(die_at)); + auto backend = startVictim(); + seed("p/roots/victim/loose_file"); + /// `BAD_ARGUMENTS` is a deterministic local failure to the request engine: the attempt is not + /// reissued and not resolved by a read, so the landed write surfaces as a refusal -- the shape of a + /// process that died right after its write landed. + const auto died_with = [](const char * what) + { + return std::make_exception_ptr(DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "{}", what)); + }; + bool died = false; + switch (die_at) + { + case DieAt::AfterTombstone: + backend->onWriteCommitted(layout.ownerKey("victim"), [&] + { + if (!std::exchange(died, true)) + std::rethrow_exception(died_with("died after the tombstone")); + }); + break; + case DieAt::AfterFarewell: + /// The farewell is written by `~Pool`, which swallows a throw; die on the verify read instead. + backend->onWriteCommitted(layout.mountKey("victim"), [&] + { + if (slot().isCleanRelease() && !std::exchange(died, true)) + backend->failNextReadWith(layout.ownerKey("victim"), died_with("died after the farewell")); + }); + break; + case DieAt::AfterMark: + backend->onWriteCommitted(layout.ownerKey("victim"), [&] + { + if (owner().retirement_verified && !std::exchange(died, true)) + std::rethrow_exception(died_with("died after the mark")); + }); + break; + } + + DecommissionReport first; + bool threw = false; + try + { + first = run("a1"); + } + catch (const DB::Exception &) + { + threw = true; + } + ASSERT_TRUE(died); + switch (die_at) + { + case DieAt::AfterTombstone: + EXPECT_TRUE(threw) << "the tombstone write is refused after it landed; nothing else ran"; + break; + case DieAt::AfterFarewell: + EXPECT_FALSE(threw) << "a failed verify read is reported, not thrown"; + EXPECT_FALSE(first.member_retired); + break; + case DieAt::AfterMark: + EXPECT_FALSE(threw) << "an unclear mark write is settled by the owner re-read"; + EXPECT_TRUE(first.member_retired) << "the re-read finds the verified mark"; + break; + } + expectOwner(die_at == DieAt::AfterMark ? 3 : 2, true, die_at == DieAt::AfterMark); + + const auto rerun = run("a2"); + EXPECT_TRUE(rerun.warnings.empty()) << fmt::format("{}", fmt::join(rerun.warnings, "; ")); + EXPECT_TRUE(rerun.member_retired); + expectOwner(die_at == DieAt::AfterMark ? 5 : 4, true, true); + EXPECT_FALSE(exists("p/roots/victim/loose_file")); + } +} + +namespace +{ + +/// The first roots delete under the victim lets the admin's local lease deadline pass (the fake clock jumps a +/// TTL); every delete admitted on the mount plane after it is refused. Optionally also GC-fences the body. +class ExpireLeaseOnFirstRootsDeleteBackend : public InMemoryBackend +{ +public: + std::shared_ptr clock; + uint64_t jump_ms = 0; + bool fence_body = false; + bool fired() const { return done; } + + RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override + { + if (!done && key.starts_with("p/roots/victim/")) + { + done = true; + clock->now += jump_ms; + if (fence_body) + { + const auto current = InMemoryBackend::read(mount_key, access); + const MountLease body = decodeMountLease(current->bytes).fencedCopy(); + if (!InMemoryBackend::write(mount_key, encodeMountLease(body), current->value, access)) + throw std::runtime_error("expire fixture: the fence conflicted"); + } + } + return InMemoryBackend::remove(key, expected_value, access); + } + +private: + inline static const String mount_key = "p/gc/server-roots/victim/mount"; + bool done = false; +}; + +/// On the staging LIST (the request before the roots LIST) lets the admin's local lease deadline pass and +/// GC-fences its body in the store: the fence loss then surfaces on the roots LIST page, not on a delete. +class ExpireLeaseBeforeRootsListBackend : public InMemoryBackend +{ +public: + using Backend::list; + std::shared_ptr clock; + uint64_t jump_ms = 0; + bool fired() const { return done; } + + RawListPage list(const String & prefix, const String & cursor, size_t limit, TransportAccess & access) override + { + RawListPage page = InMemoryBackend::list(prefix, cursor, limit, access); + if (!done && prefix == "p/staging/victim/") + { + done = true; + clock->now += jump_ms; + const auto current = InMemoryBackend::read(mount_key, access); + const MountLease body = decodeMountLease(current->bytes).fencedCopy(); + if (!InMemoryBackend::write(mount_key, encodeMountLease(body), current->value, access)) + throw std::runtime_error("expire-before-list fixture: the fence conflicted"); + } + return page; + } + +private: + inline static const String mount_key = "p/gc/server-roots/victim/mount"; + bool done = false; +}; + +/// A store whose `roots/` objects carry content-derived ETags (as S3 does). While `capture` is set, every +/// delete under the victim's prefixes applies and also leaves a copy of the attempt in flight, which the test +/// delivers later with `replay`. +class LateDeleteBackend : public InMemoryBackend +{ +public: + struct Late { String key; Etag etag; }; + std::vector late; + bool capture = false; + + std::optional read(const String & key, TransportAccess & access) override + { + auto raw = InMemoryBackend::read(key, access); + if (raw && isRoots(key)) + raw->value = contentValue(raw->bytes); + return raw; + } + + std::optional head(const String & key, TransportAccess & access) override + { + auto meta = InMemoryBackend::head(key, access); + if (meta && isRoots(key)) + if (const auto raw = InMemoryBackend::read(key, access)) + meta->value = contentValue(raw->bytes); + return meta; + } + + RawListPage list(const String & prefix, const String & cursor, size_t limit, TransportAccess & access) override + { + RawListPage page = InMemoryBackend::list(prefix, cursor, limit, access); + for (RawListedKey & listed : page.keys) + if (listed.value && isRoots(listed.key)) + if (const auto raw = InMemoryBackend::read(listed.key, access)) + listed.value = contentValue(raw->bytes); + return page; + } + + std::expected write(const String & key, const String & bytes, + const std::optional & expected_value, TransportAccess & access) override + { + if (!isRoots(key)) + return InMemoryBackend::write(key, bytes, expected_value, access); + std::optional minted_expected; + if (expected_value) + { + const auto current = InMemoryBackend::read(key, access); + if (!current || contentValue(current->bytes) != *expected_value) + return std::unexpected(RawConflict{}); + minted_expected = current->value; + } + const auto written = InMemoryBackend::write(key, bytes, minted_expected, access); + if (!written) + return written; + return contentValue(bytes); + } + + RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override + { + if (capture && isVictim(key)) + { + OperationForTest op(*this); + if (const auto meta = (*op).head(key, Retry::once())) + late.push_back({key, meta->etag}); + } + if (!isRoots(key)) + return InMemoryBackend::remove(key, expected_value, access); + const auto current = InMemoryBackend::read(key, access); + if (!current) + return RawRemoval::Gone; + if (contentValue(current->bytes) != expected_value) + return RawRemoval::Mismatch; + return InMemoryBackend::remove(key, current->value, access); + } + + /// Deliver every captured attempt now, through the ordinary exact-incarnation delete. + std::vector> replay() + { + std::vector> outcomes; + OperationForTest op(*this); + for (const Late & attempt : late) + outcomes.emplace_back(attempt.key, (*op).remove(attempt.key, attempt.etag, Retry::once())); + return outcomes; + } + +private: + static bool isRoots(const String & key) { return key.starts_with("p/roots/victim/"); } + static bool isVictim(const String & key) + { + return isRoots(key) || key.starts_with("p/staging/victim/") || key.starts_with("p/cas/manifests/victim/") + || key.starts_with("p/cas/ns/"); + } + static String contentValue(const String & bytes) { return "content-" + DB::Cas::tests::hexOf(bytes); } +}; + +/// Each delete under the victim's roots prefix takes `delay`, as a slow store would. `on_first_delete` runs +/// once, before the first of them; `after_delay` runs after each delay. +class SlowRootsDeleteBackend : public InMemoryBackend +{ +public: + std::chrono::milliseconds delay{0}; + std::function on_first_delete; + std::function after_delay; + + RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override + { + if (key.starts_with("p/roots/victim/")) + { + if (on_first_delete) + std::exchange(on_first_delete, {})(); + std::this_thread::sleep_for(delay); + if (after_delay) + after_delay(); + } + return InMemoryBackend::remove(key, expected_value, access); + } +}; + +/// The tombstone write lands, GC fences the admin's body, and the write's answer is lost; the owner then +/// cannot be read for `unreadable_ms` of the fake clock, longer than the write's own retry window, so the +/// write gives up after sending. +class OwnerUnreadableAfterTombstoneBackend : public InMemoryBackend +{ +public: + std::shared_ptr clock; + std::atomic unreadable_ms = 0; + bool landed() const { return landed_at.load() != 0; } + + std::expected write(const String & key, const String & bytes, + const std::optional & expected_value, TransportAccess & access) override + { + if (!landed() && key == owner_key && decodeOwner(bytes).retired_at_ms) + { + landed_at = clock->now.load(); + const auto current = InMemoryBackend::read(mount_key, access); + const MountLease body = decodeMountLease(current->bytes).fencedCopy(); + if (!InMemoryBackend::write(mount_key, encodeMountLease(body), current->value, access)) + throw std::runtime_error("unreadable-owner fixture: the fence conflicted"); + injectAmbiguousLandedWrite(key); + } + return InMemoryBackend::write(key, bytes, expected_value, access); + } + + std::optional read(const String & key, TransportAccess & access) override + { + if (landed() && key == owner_key && clock->now.load() < landed_at.load() + unreadable_ms.load()) + throw Poco::TimeoutException("unreadable-owner fixture: the owner does not answer"); + return InMemoryBackend::read(key, access); + } + +private: + inline static const String mount_key = "p/gc/server-roots/victim/mount"; + inline static const String owner_key = "p/gc/server-roots/victim/owner"; + std::atomic landed_at = 0; +}; + +} + +TEST_F(CASDecommission, DrainStopsOnTheAdminLeaseFence) +{ + auto backend = startVictim(std::make_shared()); + seed("p/roots/victim/a_first"); + seed("p/roots/victim/b_second"); + auto clock = farClock(); + backend->clock = clock; + backend->jump_ms = 31'000; /// past the default 30 s TTL + + const auto report = runOnClock("a1", clock); + + ASSERT_TRUE(backend->fired()); + EXPECT_FALSE(report.member_retired); + ASSERT_FALSE(report.warnings.empty()); + EXPECT_EQ(report.mountpoint_objects_removed, 1u) << "the first delete was already sent"; + EXPECT_TRUE(exists("p/roots/victim/b_second")) + << "a delete admitted after the lease deadline is refused locally"; + expectOwner(2, true, false); +} + +/// After the first delete refused by the fence the walk stops: one warning, the later objects untouched. +TEST_F(CASDecommission, DrainStopsWalkingAfterTheFirstRefusedDelete) +{ + auto backend = startVictim(std::make_shared()); + for (const char * name : {"a_first", "b_second", "c_third"}) + seed(String("p/roots/victim/") + name); + auto clock = farClock(); + backend->clock = clock; + backend->jump_ms = 31'000; + + const auto report = runOnClock("a1", clock); + + ASSERT_TRUE(backend->fired()); + const auto refused = std::count_if(report.warnings.begin(), report.warnings.end(), + [](const String & warning) { return warning.starts_with("decommission drain: "); }); + EXPECT_EQ(refused, 1) << fmt::format("{}", fmt::join(report.warnings, "; ")); + EXPECT_TRUE(exists("p/roots/victim/c_third")); +} + +/// The admin is fenced in the store AND past its local deadline after its first roots delete: the drain stops with +/// data already gone, so the tombstone stays and the member cannot start over the hole. +TEST_F(CASDecommission, FenceDuringTheDrainKeepsTheTombstone) +{ + auto backend = startVictim(std::make_shared()); + seed("p/roots/victim/a_first"); + seed("p/roots/victim/b_second"); + auto clock = farClock(); + backend->clock = clock; + backend->jump_ms = 31'000; + backend->fence_body = true; + + const auto report = runOnClock("a1", clock); + + ASSERT_TRUE(backend->fired()); + EXPECT_FALSE(report.member_retired); + EXPECT_TRUE(std::ranges::any_of(report.warnings, [](const String & w) { return w.contains("lost during the drain"); })) + << fmt::format("{}", fmt::join(report.warnings, "; ")); + EXPECT_FALSE(exists("p/roots/victim/a_first")) << "the first delete landed"; + expectOwner(2, true, false); /// claim 1, tombstone 2, kept + EXPECT_TRUE(slot().gc_fenced); + /// A start with a server uuid would rejoin a verified retirement; this one is not verified. + expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "decommission is not finished", [&] + { Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_id = UInt128(7), .server_root_id = "victim"}); }); +} + +TEST_F(CASDecommission, FenceLossOnAListPageStillRunsTheConfirm) +{ + auto backend = startVictim(std::make_shared()); + auto clock = farClock(); + backend->clock = clock; + backend->jump_ms = 31'000; + + EXPECT_THROW(runOnClock("a1", clock), DB::Exception); + + ASSERT_TRUE(backend->fired()); + expectOwner(2, true, false); /// the exception exit confirmed and kept the tombstone: the drain had started +} + +/// GC fences the admin right after it dropped a namespace, and its lease deadline passes. The run stops with that +/// namespace gone: the tombstone stays, a restart of the member is refused, and a rerun retires it. +TEST_F(CASDecommission, FenceAfterANamespaceDropKeepsTheTombstone) +{ + auto backend = startVictim(std::make_shared(), [](Pool & victim) + { + makeTableWithRefs(victim, "victim/db/t1", 1, 0); + makeTableWithRefs(victim, "victim/db/t2", 1, 0); + }); + auto clock = farClock(); + bool fenced = false; + std::optional report; + std::optional error_code; + try + { + report = decommissionPoolMember(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "a1"}, "victim", + [&](const CasEvent & e) + { + if (e.type != CasEventType::MemberDecommission || e.outcome != "namespace_removed" + || std::exchange(fenced, true)) + return; + fenceSlot(); + clock->now += 31'000; /// past the default 30 s TTL: the admin sends nothing more + }, + /*request_gc_round=*/{}, [clock] { return clock->nowFn()(); }, [clock](uint64_t ms) { clock->sleepFn()(ms); }); + } + catch (const DB::Exception & e) + { + error_code = e.code(); /// the second namespace drop is refused by the fence + } + + ASSERT_TRUE(fenced); + ASSERT_NE(report.has_value(), error_code.has_value()) << "the run either throws or reports"; + if (report) + EXPECT_FALSE(report->member_retired); + expectOwner(2, true, false); + /// A start with a server uuid would rejoin a verified retirement; this one is not verified. + expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "decommission is not finished", [&] + { Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_id = UInt128(7), .server_root_id = "victim"}); }); + + const auto rerun = run("a2"); + EXPECT_TRUE(rerun.member_retired) << fmt::format("{}", fmt::join(rerun.warnings, "; ")); +} + +/// Real time: the background renewer keeps the admin's lease alive across a drain longer than one TTL (1 s +/// TTL, renewal every 200 ms, three roots deletes of 400 ms each), and the run retires the member. +TEST_F(CASDecommission, LongDrainOutlivesOneTtl) +{ + auto backend = startVictim(std::make_shared()); + backend->delay = std::chrono::milliseconds(400); + for (const char * name : {"a", "b", "c"}) + seed(String("p/roots/victim/") + name); + /// A stop that races a renewal skips the farewell; that race is a known open defect. The last delete + /// waits for a renewal to land and then half a renewal period, so the stop lands between two renewals. + struct MountWrites + { + std::mutex mutex; + std::condition_variable landed; + size_t count = 0; + }; + auto mount_writes = std::make_shared(); + backend->onWriteCommitted(layout.mountKey("victim"), [mount_writes] + { + std::lock_guard lock(mount_writes->mutex); + ++mount_writes->count; + mount_writes->landed.notify_all(); + }); + auto deletes = std::make_shared(0); + backend->after_delay = [mount_writes, deletes] + { + if (++*deletes != 3) + return; + std::unique_lock lock(mount_writes->mutex); + const size_t seen = mount_writes->count; + ASSERT_TRUE(mount_writes->landed.wait_for(lock, std::chrono::seconds(5), [&] { return mount_writes->count > seen; })); + lock.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + }; + + const auto started = std::chrono::steady_clock::now(); + const auto report = run("a1", realTimeConfig()); + + EXPECT_GE(std::chrono::steady_clock::now() - started, std::chrono::milliseconds(1200)); + EXPECT_EQ(report.mountpoint_objects_removed, 3u); + EXPECT_TRUE(report.warnings.empty()) << fmt::format("{}", fmt::join(report.warnings, "; ")); + EXPECT_TRUE(report.member_retired); +} + +/// Once the owner is tombstoned, the admin's own self-remount fails on the owner check and never reclaims +/// under a fresh epoch. +TEST_F(CASDecommission, FencedAdminFailsInsteadOfRemounting) +{ + auto backend = startVictim(); + auto admin = Pool::openForDecommission(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "a1"}, "victim"); + const uint64_t admin_epoch = admin->writerEpoch(); + tombstoneOwner(); + fenceSlot(); + admin->tripMountLost(); /// what the renewer does when it finds its body fenced + DB::Cas::tests::ScopedLoggerCapture capture("CasPool", "warning"); + EXPECT_FALSE(admin->tryRemountOnce()); + EXPECT_EQ(admin->lifecycle(), PoolLifecycle::IdentityLost); + EXPECT_NE(capture.captured().find("lost its admin claim and does not remount"), String::npos) << capture.captured(); + const MountLease body = slot(); + EXPECT_EQ(body.writer_epoch, admin_epoch); + EXPECT_TRUE(body.gc_fenced); + EXPECT_FALSE(admin->mayMutate()); +} + +/// The admin is GC-fenced during its drain; its renewer trips and its self-remount fails +/// (owner retired). The run's exit confirms, finds the claim lost and keeps the tombstone. Real time: +/// TTL 1 s, renewal every 200 ms, a 400 ms delete per roots object. +TEST_F(CASDecommission, FailedRemountStillRunsTheConfirm) +{ + auto backend = startVictim(std::make_shared()); + backend->delay = std::chrono::milliseconds(400); + for (const char * name : {"a", "b", "c", "d"}) + seed(String("p/roots/victim/") + name); + bool fenced = false; + backend->on_first_delete = [&] + { + fenced = true; + fenceSlot(); + }; + const uint64_t remount_failures_before = ProfileEvents::global_counters[ProfileEvents::CASRemountFailed]; + + const auto report = run("a1", realTimeConfig()); + + ASSERT_TRUE(fenced); + EXPECT_FALSE(report.member_retired); + EXPECT_GT(ProfileEvents::global_counters[ProfileEvents::CASRemountFailed], remount_failures_before) + << "the admin's self-remount ran and failed"; + expectOwner(2, true, false); /// kept: the claim was lost during the drain + EXPECT_FALSE(exists("p/roots/victim/a")) << "the delete that tripped the fence landed"; + EXPECT_TRUE(slot().gc_fenced); +} + +/// A paused run resumes after a rerun refreshed the tombstone: its rollback names an older incarnation and +/// cannot remove the rerun's tombstone. +TEST_F(CASDecommission, PausedRunCannotUntombstoneARerunsTombstone) +{ + auto backend = startVictim(); + std::optional rerun; + bool armed = true; + backend->onWriteCommitted(layout.ownerKey("victim"), [&] + { + if (!std::exchange(armed, false)) + return; + /// Run 1 is paused right after its tombstone. GC fences it; the rerun reclaims, refreshes the + /// tombstone, drains, farewells and verifies. + fenceSlot(); + rerun = run("a2"); + }); + + const auto first = run("a1"); + + ASSERT_TRUE(rerun.has_value()); + EXPECT_TRUE(rerun->member_retired); + EXPECT_FALSE(first.member_retired); + EXPECT_EQ(first.warnings.size(), 1u) << fmt::format("{}", fmt::join(first.warnings, "; ")); + expectOwner(4, true, true); /// run 1 tombstone 2, rerun tombstone 3, rerun mark 4 +} + +/// A delete attempt sent by a run and delivered after a rejoin can remove only an object whose key and bytes +/// repeat: the roots objects. Namespace, manifest and staging keys of the new member never match. +TEST_F(CASDecommission, InFlightDeleteAfterRejoinHitsOnlyIdenticalRootsBytes) +{ + auto backend = startVictim(std::make_shared(), oneTable); /// no debris: it would block the mark + const RootNamespace ns("victim/db/t1"); + seed("p/staging/victim/3/old.tmp"); + { + OperationForTest seed_op(*backend); + (*seed_op).create("p/roots/victim/clickhouse_access_check", "probe", Retry::once()); + } + backend->capture = true; + const auto report = run("a1"); + backend->capture = false; + ASSERT_TRUE(report.member_retired); + ASSERT_FALSE(backend->late.empty()); + + /// The rejoin and the new member's first writes, by hand: the next owner life, a new staging object, a + /// new manifest, and the access probe with the same bytes under the same key. + { + OperationForTest op(*backend); + const auto current = (*op).read(layout.ownerKey("victim"), Retry::once()); + const OwnerObject rejoined = nextOwnerLife(decodeOwner(current->bytes), std::nullopt, false); + ASSERT_TRUE(std::holds_alternative( + (*op).replace(layout.ownerKey("victim"), encodeOwner(rejoined), current->etag, Retry::once()))); + ASSERT_TRUE(std::holds_alternative((*op).create("p/staging/victim/9/new.tmp", "y", Retry::once()))); + ASSERT_TRUE(std::holds_alternative((*op).create("p/roots/victim/clickhouse_access_check", "probe", Retry::once()))); + } + const ManifestRef new_ref{.writer_epoch = 9, .build_sequence = 1, .manifest_ordinal = 1}; + const ManifestId new_manifest = writeManifestRaw(*backend, layout, ns, new_ref, {}); + + backend->replay(); + EXPECT_FALSE(exists("p/roots/victim/clickhouse_access_check")) + << "identical bytes repeat the ETag: the late delete lands on the new member's probe (the documented residual)"; + EXPECT_TRUE(exists("p/staging/victim/9/new.tmp")); + EXPECT_TRUE(exists(layout.manifestKey(new_manifest))); +} + +/// The tombstone lands and the engine throws before reporting it, while GC fences the admin: the exit reads +/// the owner back, finds this run's tombstone, sees the lost claim and rolls the tombstone back. +TEST_F(CASDecommission, TombstoneLandedBehindAnErrorIsConfirmed) +{ + auto backend = startVictim(); + bool landed = false; + backend->onWriteCommitted(layout.ownerKey("victim"), [&] + { + if (std::exchange(landed, true)) + return; + fenceSlot(); + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "the tombstone landed and its answer was lost"); + }); + + EXPECT_THROW(run("a1"), DB::Exception); + + ASSERT_TRUE(landed); + expectOwner(3, false, false); /// claim 1, tombstone 2, rollback 3 +} + +/// The tombstone write gives up after sending; the owner answers again later and holds this run's tombstone. +/// GC fenced the admin, so the exit rolls the tombstone back. +TEST_F(CASDecommission, TombstoneWriteThatGaveUpAfterSendingIsConfirmed) +{ + auto backend = startVictim(std::make_shared()); + auto clock = farClock(); + backend->clock = clock; + backend->unreadable_ms = 100'000; /// longer than the tombstone write retries + + EXPECT_THROW(runOnClock("a1", clock), DB::Exception); + + ASSERT_TRUE(backend->landed()); + backend->unreadable_ms = 0; + expectOwner(3, false, false); /// claim 1, tombstone 2, rollback 3 +} + +/// A later run tombstones the owner between this run's farewell and its verify. The owner is past this run's +/// tombstone but unverified: this run reports rerun. +TEST_F(CASDecommission, LaterUnverifiedTombstoneIsNotARetirement) +{ + auto backend = startVictim(); + bool refreshed = false; + backend->onWriteCommitted(layout.mountKey("victim"), [&] + { + if (slot().isCleanRelease() && !std::exchange(refreshed, true)) + tombstoneOwner(); + }); + + const auto report = run("a1"); + + ASSERT_TRUE(refreshed); + EXPECT_FALSE(report.member_retired); + EXPECT_EQ(report.warnings.size(), 2u) << fmt::format("{}", fmt::join(report.warnings, "; ")); + expectOwner(3, true, false); +} + +/// The owner moves on to a later, unretired life between this run's farewell and its verify (a rejoin, or +/// another run's rollback). Only a verified mark is a retirement: the run reports 0 and says the owner moved. +TEST_F(CASDecommission, OwnerThatMovedOnIsNotReportedRetired) +{ + auto backend = startVictim(); + bool moved = false; + backend->onWriteCommitted(layout.mountKey("victim"), [&] + { + if (!slot().isCleanRelease() || std::exchange(moved, true)) + return; + OperationForTest op(*backend); + const auto current = (*op).read(layout.ownerKey("victim"), Retry::once()); + ASSERT_TRUE(std::holds_alternative((*op).replace(layout.ownerKey("victim"), + encodeOwner(nextOwnerLife(decodeOwner(current->bytes), std::nullopt, false)), current->etag, Retry::once()))); + }); + + const auto report = run("a1"); + + ASSERT_TRUE(moved); + EXPECT_FALSE(report.member_retired); + ASSERT_EQ(report.warnings.size(), 2u) << fmt::format("{}", fmt::join(report.warnings, "; ")); + EXPECT_NE(report.warnings.back().find("moved on to life 3"), String::npos) << report.warnings.back(); + EXPECT_FALSE(owner().retired_at_ms.has_value()); +} + +TEST_F(CASDecommission, RetirementLogsTheUuidAndLife) +{ + auto backend = startVictim(); + DB::Cas::tests::ScopedLoggerCapture capture("CasDecommission", "information"); + + const auto report = run("a1"); + + ASSERT_TRUE(report.member_retired); + const String expected = "retired server_uuid=" + u128ToHex(owner().server_uuid) + " at owner life 2"; + EXPECT_NE(capture.captured().find(expected), String::npos) << capture.captured(); +} + +/// The owner moves under the mark write: the run reports why the mark is missing, not only that it is. +TEST_F(CASDecommission, MarkWriteConflictIsReported) +{ + auto backend = startVictim(); + bool farewell_landed = false; + backend->onWriteCommitted(layout.mountKey("victim"), [&] + { + if (slot().isCleanRelease()) + farewell_landed = true; + }); + bool moved = false; + backend->onBeforeWrite(layout.ownerKey("victim"), [&] + { + if (!farewell_landed || std::exchange(moved, true)) + return; + OperationForTest op(*backend); + const auto current = (*op).read(layout.ownerKey("victim"), Retry::once()); + ASSERT_TRUE(std::holds_alternative( + (*op).replace(layout.ownerKey("victim"), current->bytes, current->etag, Retry::once()))); + }); + + const auto report = run("a1"); + + ASSERT_TRUE(moved); + EXPECT_FALSE(report.member_retired); + const auto mark_warning = std::find_if(report.warnings.begin(), report.warnings.end(), + [](const String & warning) { return warning.starts_with("the verified mark was not written"); }); + EXPECT_NE(mark_warning, report.warnings.end()) << fmt::format("{}", fmt::join(report.warnings, "; ")); + EXPECT_FALSE(owner().retirement_verified); +} + +namespace +{ + +/// Once armed, the next catalog read fails, and so does the slot read after it: a drain that fails on a store +/// outage, followed by a confirm that fails for the same reason. +class DrainThenConfirmFailBackend : public InMemoryBackend +{ +public: + void arm() { armed = true; } + bool confirmFailed() const { return slot_failed; } + + std::optional read(const String & key, TransportAccess & access) override + { + if (armed && !catalog_failed && key == catalog_key) + { + catalog_failed = true; + throw std::runtime_error("injected: the catalog does not answer"); + } + if (catalog_failed && !slot_failed && key == mount_key) + { + slot_failed = true; + throw std::runtime_error("injected: the slot does not answer"); + } + return InMemoryBackend::read(key, access); + } - EXPECT_TRUE(report.warnings.empty()); - EXPECT_TRUE(report.slot_removed); - OperationForTest raw_op(*backend); - EXPECT_FALSE((*raw_op).head("p/gc/server-roots/victim/mount", Retry::once()).has_value()); - EXPECT_FALSE((*raw_op).head("p/gc/server-roots/victim/epoch", Retry::once()).has_value()); - const auto owner = (*raw_op).read("p/gc/server-roots/victim/owner", Retry::once()); - ASSERT_TRUE(owner.has_value()); - EXPECT_TRUE(decodeOwner(owner->bytes).retired_at_ms.has_value()); -} - -/// One decommission command spans several requests on the SAME open-fence engine: the -/// pre-impersonation catalog cut (before any `Pool` exists), the namespace drop's own catalog re-read, -/// and -- once the row it dropped is no longer owned -- the full retirement tail's reads, deletes and -/// final owner tombstone, all issued after `admin.reset()` destroys the `Pool`. Catalog-row deletion -/// is GC's job (`dropNamespace` only reaches `Removing`), so this is necessarily two commands: the -/// first proves the drop and its catalog read landed, the second (after GC folds the row) proves the -/// retirement tail's requests landed past the `Pool`'s own lifetime. -TEST(CASDecommission, RunsOnAnOpenFence) +private: + inline static const String catalog_key = Layout("p").refCatalogKey(); + inline static const String mount_key = "p/gc/server-roots/victim/mount"; + bool armed = false; + bool catalog_failed = false; + bool slot_failed = false; +}; + +} + +/// The drain fails and the confirm's own read fails too: the drain's error is the one that propagates, and +/// the failed confirm is logged. +TEST_F(CASDecommission, FailedConfirmKeepsTheDrainError) { - auto backend = std::make_shared(); + auto backend = startVictim(std::make_shared()); + backend->onWriteCommitted(layout.ownerKey("victim"), [&] { backend->arm(); }); + DB::Cas::tests::ScopedLoggerCapture capture("CasDecommission", "warning"); + + try { - auto victim = openVictim(backend); - makeTableWithRefs(*victim, "victim/db/t1", 1, 0); + run("a1"); + ADD_FAILURE() << "the drain must fail"; + } + catch (const std::exception & e) + { + EXPECT_NE(String(e.what()).find("the catalog does not answer"), String::npos) << e.what(); } - const auto pending = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); - EXPECT_EQ(pending.namespaces_removed, 1u); - EXPECT_FALSE(pending.slot_removed); - EXPECT_FALSE(pending.warnings.empty()); - - drainCompletedNamespaceRemovals(backend); + ASSERT_TRUE(backend->confirmFailed()); + EXPECT_NE(capture.captured().find("could not confirm the admin claim"), String::npos) << capture.captured(); +} - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin2"}, "victim"); - EXPECT_TRUE(report.warnings.empty()); - EXPECT_TRUE(report.slot_removed); - OperationForTest raw_op(*backend); - EXPECT_FALSE((*raw_op).head("p/gc/server-roots/victim/mount", Retry::once()).has_value()); - EXPECT_FALSE((*raw_op).head("p/gc/server-roots/victim/epoch", Retry::once()).has_value()); - const auto owner = (*raw_op).read("p/gc/server-roots/victim/owner", Retry::once()); - ASSERT_TRUE(owner.has_value()); - EXPECT_TRUE(decodeOwner(owner->bytes).retired_at_ms.has_value()); +TEST_F(CASDecommission, OpenLogsTheSkippedProbe) +{ + auto backend = startVictim(); + DB::Cas::tests::ScopedLoggerCapture capture("CasPool", "information"); + auto admin = Pool::openForDecommission(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "a1"}, "victim"); + EXPECT_NE(capture.captured().find("capability probe was skipped"), String::npos) << capture.captured(); } -TEST(CASDecommission, SuccessfulDecommissionLeavesTombstonedOwnerAnchor) +/// The admin impersonates the owner it retires, so whose life is "ours" is ambiguous: it never takes another +/// uuid's terminal body and refuses before writing anything. +TEST_F(CASDecommission, AdminDoesNotReclaimAForeignTerminalBody) { - auto backend = std::make_shared(); - { auto victim = openVictim(backend); } + startVictim(); + { + OperationForTest op(*victim_backend); + const auto got = (*op).read(layout.mountKey("victim"), Retry::once()); + MountLease body = decodeMountLease(got->bytes); + body.server_uuid = UInt128(5); + body.seq += 1; + ASSERT_TRUE(std::holds_alternative( + (*op).replace(layout.mountKey("victim"), encodeMountLease(body), got->etag, Retry::once()))); + } + const OwnerObject owner_before = owner(); + const MountLease slot_before = slot(); - const String owner_key = "p/gc/server-roots/victim/owner"; - OperationForTest raw_op(*backend); - const auto before = (*raw_op).read(owner_key, Retry::once()); - ASSERT_TRUE(before.has_value()); - EXPECT_FALSE(decodeOwner(before->bytes).retired_at_ms.has_value()); + expectThrowsCodeWithMessage(ErrorCodes::ABORTED, "alive or contended", [&] { run("admin"); }); - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + expectOwner(owner_before.owner_life, /*retired=*/false, /*verified=*/false); + EXPECT_EQ(slot().seq, slot_before.seq); +} - EXPECT_TRUE(report.warnings.empty()); - EXPECT_TRUE(report.slot_removed); - const auto after = (*raw_op).read(owner_key, Retry::once()); - ASSERT_TRUE(after.has_value()); - EXPECT_NE(after->etag, before->etag); - EXPECT_EQ(decodeOwner(after->bytes).server_uuid, decodeOwner(before->bytes).server_uuid); - EXPECT_TRUE(decodeOwner(after->bytes).retired_at_ms.has_value()); +namespace +{ +/// A rejoin needs a real server uuid: a start without one (uuid 0) never rejoins. +const UInt128 victim_uuid = UInt128(7); + +PoolConfig victimConfig(UInt128 uuid = victim_uuid) +{ + return PoolConfig{.pool_prefix = "p", .server_id = uuid, .server_root_id = "victim"}; } -TEST(CASDecommission, SuccessorOwnerRewriteWinsBeforeTombstone) +/// One completed decommission of the victim with nothing left for GC: the state a rejoin starts from. +void retireVictim(const std::shared_ptr & backend, const String & admin_srid) { - auto backend = std::make_shared(); - { auto victim = openVictim(backend); } - backend->armForSuccessorRewrite(); + const auto report = decommissionPoolMember(backend, PoolConfig{.pool_prefix = "p", .server_root_id = admin_srid}, "victim"); + if (!report.member_retired || report.namespaces_pending_gc != 0) + throw std::runtime_error("fixture: the decommission did not retire the victim cleanly"); +} - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); +bool victimSubtreeEmpty(Backend & backend) +{ + const Layout layout("p"); + CasRequests requests = openRequestsForTest(backend); + CasOperation op = requests.admit(); + return serverRootSubtreeEmpty(op, layout, "victim", CasRefCatalog::read(op, layout).catalog); +} - ASSERT_TRUE(backend->successorInjected()); - EXPECT_FALSE(report.slot_removed); - ASSERT_EQ(report.warnings.size(), 1u); - EXPECT_NE(report.warnings.front().find("successor reclaimed"), String::npos); +/// GC removes the dropped namespaces' catalog rows and manifests over a few rounds; a rejoin waits for all of them. +void gcUntilVictimSubtreeEmpty(const std::shared_ptr & backend) +{ + auto store = Pool::open(backend, PoolConfig{ + .pool_prefix = "p", .server_root_id = "gc", .gc_fold_threshold = 1, .gc_fold_max_defer_rounds = 0}); + Gc gc(store, UInt128{991}); + for (int round = 0; round < 8 && !victimSubtreeEmpty(*backend); ++round) + runRegularRoundReclaiming(gc); + if (!victimSubtreeEmpty(*backend)) + throw std::runtime_error("fixture: GC left data of the victim's previous life"); +} - OperationForTest raw_op(*backend); - const auto owner = (*raw_op).read("p/gc/server-roots/victim/owner", Retry::once()); - ASSERT_TRUE(owner.has_value()); - EXPECT_EQ(PersistedEtag::capture(owner->etag).value, backend->successorOwnerValue()); - EXPECT_EQ(owner->bytes, backend->successorOwnerBytes()); - EXPECT_FALSE(decodeOwner(owner->bytes).retired_at_ms.has_value()); -} - -/// Final whole-branch review finding (Important): -/// an ambiguous outcome on the owner tombstone write -- the write lands but its response is lost, the -/// same shape as a real SDK timeout after a landed write -- must not be reported as a hard failure. -/// `op.replace`'s own resolve read settles this (current bytes already match the intended tombstone) -/// and reports `Committed`. `injectAmbiguousLandedWrite` throws `Poco::TimeoutException` after -/// applying the write: the engine's write loop treats a `Poco::Exception` as a transport fault (never -/// a caller bug), which is exactly the class this scenario models -- a bare `std::runtime_error` here -/// would propagate unchanged instead of being resolved. -TEST(CASDecommission, OwnerTombstoneAmbiguousSuccessResolvesToCommitted) +/// The rejoined member writes a part under the old life's namespace name; GC rounds run from another member +/// keep it, and the member's slot stays its own. +void expectWriteSurvivesGc(const std::shared_ptr & backend, const PoolPtr & member, UInt128 member_uuid) { - auto backend = std::make_shared(); - { auto victim = openVictim(backend); } - backend->injectAmbiguousLandedWrite("p/gc/server-roots/victim/owner"); + const RootNamespace ns{"victim/db/t1"}; + const String payload = "written after the rejoin"; + PartWriteInfo info; + info.intended_ref = ns.string() + "/r"; + auto build = member->beginPartWrite(info); + ManifestEntry entry; + entry.path = "data.bin"; + entry.placement = EntryPlacement::Blob; + entry.ref = idOf(payload); + entry.blob_size = payload.size(); + const ManifestId manifest = build->stageManifest({entry}); + build->precommitAdd(ns, "r", manifest); + build->putBlob(idOf(payload), BlobSource::fromString(payload)); + build->promote(ns, "r", build->buildId(), manifest); - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + { + auto gc_store = Pool::open(backend, PoolConfig{ + .pool_prefix = "p", .server_root_id = "gc", .gc_fold_threshold = 1, .gc_fold_max_defer_rounds = 0}); + Gc gc(gc_store, UInt128{992}); + for (int round = 0; round < 3; ++round) + { + runRegularRoundReclaiming(gc); + member->renewWatermarkOnce(); + } + } - EXPECT_TRUE(report.slot_removed) << "the ambiguous write actually landed and must resolve to Committed"; - EXPECT_TRUE(report.warnings.empty()); + const Layout layout("p"); + EXPECT_EQ(member->listRefs(ns).size(), 1u); + OperationForTest op(*backend); + EXPECT_TRUE((*op).head(layout.manifestKey(manifest), Retry::once()).has_value()); + EXPECT_TRUE((*op).head(layout.blobKey(idOf(payload)), Retry::once()).has_value()); + const MountLease slot = readSlot(*backend, layout, "victim"); + EXPECT_TRUE(slot.isHeldBy(member_uuid, member->writerEpoch())); + EXPECT_FALSE(slot.gc_fenced); +} - OperationForTest raw_op(*backend); - const auto owner = (*raw_op).read("p/gc/server-roots/victim/owner", Retry::once()); - ASSERT_TRUE(owner.has_value()); - EXPECT_TRUE(decodeOwner(owner->bytes).retired_at_ms.has_value()); +/// The slot body rewritten by hand; `edit` changes the decoded body before it is written back. +void rewriteSlot(Backend & backend, const Layout & layout, const std::function & edit) +{ + OperationForTest op(backend); + const auto got = (*op).read(layout.mountKey("victim"), Retry::once()); + ASSERT_TRUE(got.has_value()); + MountLease body = decodeMountLease(got->bytes); + edit(body); + body.seq += 1; + ASSERT_TRUE(std::holds_alternative( + (*op).replace(layout.mountKey("victim"), encodeMountLease(body), got->etag, Retry::once()))); +} } -/// Delegates every op to `inner`, except `remove`: while `armed`, any key starting with `fail_prefix` -/// throws an injected transient failure instead of deleting -- models a real backend transiently -/// failing to delete under one whole prefix. `disarm()` clears the failure (the resume half of -/// `FailedDrainKeepsSlotThenResumes`). Forwards every pure-virtual `Backend` member (the `CasBackend.h` -/// list) to `inner` untouched. Injects on the `remove` PRIMITIVE, not the legacy `deleteExact`: -/// `deleteListedPrefix`'s mountpoint drain (`CasDecommission.cpp`) goes through `CasOperation::remove`. -class FailDeletesUnderPrefixBackend : public Backend +TEST(CASRejoin, SameUuidStartsEmptyAndContinuesTheCounter) { -public: - FailDeletesUnderPrefixBackend(std::shared_ptr inner_, String fail_prefix_) - : inner(std::move(inner_)), fail_prefix(std::move(fail_prefix_)) + auto backend = std::make_shared(); { + auto victim = Pool::open(backend, victimConfig()); + makeTableWithRefs(*victim, "victim/db/t1", 2, 0); } + ASSERT_TRUE(decommissionPoolMember(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim").member_retired); + gcUntilVictimSubtreeEmpty(backend); + const Layout layout("p"); + const OwnerObject retired = readOwner(*backend, layout, "victim"); + ASSERT_TRUE(retired.retirement_verified); + const uint64_t admin_epoch = readSlot(*backend, layout, "victim").writer_epoch; + + auto rejoined = Pool::open(backend, victimConfig()); + + EXPECT_GT(rejoined->writerEpoch(), admin_epoch) << "the kept counter continues past every earlier life"; + const OwnerObject owner = readOwner(*backend, layout, "victim"); + EXPECT_EQ(owner.owner_life, retired.owner_life + 1); + EXPECT_FALSE(owner.retired_at_ms.has_value()); + EXPECT_FALSE(owner.retirement_verified); + EXPECT_TRUE(victimSubtreeEmpty(*backend)) << "the member comes back empty"; + EXPECT_EQ(readSlot(*backend, layout, "victim").writer_epoch, rejoined->writerEpoch()); + expectWriteSurvivesGc(backend, rejoined, victim_uuid); +} - void disarm() { armed = false; } +TEST(CASRejoin, NewUuidReclaimsTheOldLifesFarewell) +{ + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + const Layout layout("p"); + const MountLease farewell = readSlot(*backend, layout, "victim"); + const uint64_t retired_life = readOwner(*backend, layout, "victim").owner_life; - bool supportsListTokens() const override { return inner->supportsListTokens(); } + auto events = std::make_shared>(); + PoolConfig config = victimConfig(UInt128(77)); + config.event_sink = [events](CasEvent e) + { + if (e.type == CasEventType::MountClaim && e.outcome.starts_with("rejoin")) + events->push_back(std::move(e)); + }; + auto rejoined = Pool::open(backend, config); - /// The transport primitives forward to `inner`, except `remove`, which is what this double - /// injects through. Declared because `Backend` declares them pure. - std::optional read(const String & key, TransportAccess & access) override { return inner->read(key, access); } - std::optional head(const String & key, TransportAccess & access) override { return inner->head(key, access); } - RawListPage list(const String & prefix, const String & cursor, size_t limit, TransportAccess & access) override { return inner->list(prefix, cursor, limit, access); } - RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override + ASSERT_EQ(events->size(), 2u) << "one event per rejoin write"; + EXPECT_EQ((*events)[0].outcome, "rejoin_mount_rewrite"); + EXPECT_EQ((*events)[1].outcome, "rejoin_owner"); + for (const CasEvent & e : *events) { - if (armed && key.starts_with(fail_prefix)) - /// `Poco::TimeoutException`, the class the request engine classifies as a transport fault - /// and reissues (`Retry::standard()`): this fixture models a real backend transiently - /// failing, and `armed` stays set across every reissue of the same call, so the engine - /// reaches its own retry deadline and gives up rather than recovering on a later attempt. - throw Poco::TimeoutException("injected transient delete failure for " + key); - return inner->remove(key, expected_value, access); + EXPECT_EQ(e.detail.at("retired_uuid"), u128ToHex(victim_uuid)); + EXPECT_EQ(e.detail.at("retired_life"), std::to_string(retired_life)); + EXPECT_EQ(e.detail.at("server_uuid"), u128ToHex(UInt128(77))); + EXPECT_EQ(e.detail.at("owner_life"), std::to_string(retired_life + 1)); } - void removeManyWriteOnce(const std::vector & keys, TransportAccess & access) override { inner->removeManyWriteOnce(keys, access); } - size_t bulkDeleteKeyLimit() const override { return inner->bulkDeleteKeyLimit(); } - std::expected write(const String & key, const String & bytes, - const std::optional & expected_value, TransportAccess & access) override + + const OwnerObject owner = readOwner(*backend, layout, "victim"); + EXPECT_EQ(owner.server_uuid, UInt128(77)); + EXPECT_FALSE(owner.retired_at_ms.has_value()); + const MountLease slot = readSlot(*backend, layout, "victim"); + EXPECT_EQ(slot.server_uuid, UInt128(77)); + EXPECT_GT(slot.writer_epoch, farewell.writer_epoch); + EXPECT_EQ(slot.writer_epoch, rejoined->writerEpoch()); + expectWriteSurvivesGc(backend, rejoined, UInt128(77)); +} + +/// The old victim reclaimed the admin's farewell under a fresh epoch, saw the tombstone and fenced itself. +TEST(CASRejoin, NewUuidReclaimsASelfFencedOldVictimBody) +{ + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + const Layout layout("p"); + uint64_t victim_epoch = 0; { - return inner->write(key, bytes, expected_value, access); + OperationForTest op(*backend); + victim_epoch = allocateWriterEpoch(*op, layout, "victim", [] { return RefCatalog{}; }); } - std::unique_ptr stream(const String & key, TransportAccess & access) override { return inner->stream(key, access); } - void publish(const BlobPublishRequest & request, TransportAccess & access) override { inner->publish(request, access); } - DB::Cas::Dialect dialect() const override { return inner->dialect(); } + rewriteSlot(*backend, layout, [&](MountLease & body) + { + body.writer_epoch = victim_epoch; + body.min_active_build_sequence = 0; + body.gc_fenced = true; + }); -private: - std::shared_ptr inner; - String fail_prefix; - bool armed = true; -}; + auto rejoined = Pool::open(backend, victimConfig(UInt128(77))); + + const MountLease slot = readSlot(*backend, layout, "victim"); + EXPECT_EQ(slot.server_uuid, UInt128(77)); + EXPECT_EQ(slot.writer_epoch, rejoined->writerEpoch()); + EXPECT_FALSE(slot.gc_fenced); +} -/// Task 4 fail-close: a drain failure under the roots prefix keeps the slot terminated-but-present -/// (`report.slot_removed == false`, the mount object survives as the resume anchor). Once the fault is -/// cleared, a re-run finishes the job: the already-erased namespace is counted as -/// `namespaces_already_removed`, the leftover roots object is finally swept, and the slot is removed. -TEST(CASDecommission, FailedDrainKeepsSlotThenResumes) +TEST(CASRejoin, RefusesWhileTheDecommissionRuns) { - auto inner = std::make_shared(); + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + const Layout layout("p"); + auto rerun_admin = Pool::openForDecommission(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin2"}, "victim"); + const OwnerObject before = readOwner(*backend, layout, "victim"); + + for (const UInt128 uuid : {victim_uuid, UInt128(77)}) { - auto victim = Pool::open(inner, PoolConfig{.pool_prefix = "p", .server_root_id = "victim"}); - makeTableWithRefs(*victim, "victim/db/t1", 1, 0); + SCOPED_TRACE(u128ToHex(uuid)); + expectThrowsCodeWithMessage(ErrorCodes::ABORTED, "still running", [&] { Pool::open(backend, victimConfig(uuid)); }); } - OperationForTest raw_op(*inner); - (*raw_op).create("p/roots/victim/loose_file", "x", Retry::once()); - auto failing = std::make_shared(inner, "p/roots/victim/"); - /// The engine reissues an unresolved delete until its own retry window closes, measured on this - /// clock, so the fault (armed across every reissue) reaches a genuine give-up with no real time - /// passing -- `clock` fast-forwards through the whole 90 s `Retry::standard()` window in one call. - /// `drain_now_fn` is also this session's boot clock (decommissionPoolMember unifies the two), so the - /// admin's own mount lease needs a TTL well past that fast-forward or the farewell it attempts on - /// the way out would refuse against a deadline the retry exhaustion already ran past -- a real - /// decommission's background renewer would have kept the deadline current over 90 real seconds, but - /// nothing here advances real time to let it. - auto clock = std::make_shared(); - const auto first = decommissionPoolMember( - failing, - PoolConfig{.pool_prefix = "p", .server_root_id = "a1", .mount_lease_ttl_ms = std::chrono::milliseconds(300'000)}, - "victim", /*sink=*/{}, /*request_gc_round=*/{}, - [clock] + const OwnerObject after = readOwner(*backend, layout, "victim"); + EXPECT_EQ(after.owner_life, before.owner_life); + EXPECT_TRUE(after.retirement_verified); + EXPECT_EQ(readSlot(*backend, layout, "victim").writer_epoch, rerun_admin->writerEpoch()) << "the admin's live body is untouched"; +} + +/// Each part of the old life that can remain refuses the rejoin, names the part and writes nothing; once the +/// part is gone the start rejoins. The verified mark does not wait for GC; the rejoin does. +TEST(CASRejoin, RefusesWhileDataOfTheOldLifeRemains) +{ + const Layout layout("p"); + const auto control_objects = [&](Backend & backend) + { + OperationForTest op(backend); + std::vector etags; + for (const String & key : {layout.ownerKey("victim"), layout.epochKey("victim"), layout.mountKey("victim")}) + etags.push_back((*op).read(key, Retry::once())->etag); + return etags; + }; + for (const ServerRootLeftover leftover : + {ServerRootLeftover::CatalogRows, ServerRootLeftover::Manifests, ServerRootLeftover::RootsObjects}) + { + SCOPED_TRACE(static_cast(leftover)); + auto backend = std::make_shared(); + OperationForTest op(*backend); + /// The server uuid as `IDisk::checkAccess` formats it for the probe key and the server passes as its id. + const String old_uuid = toString(parseFromString("01234567-89ab-cdef-fedc-ba9876543210")); + String old_uuid_hex = old_uuid; + std::erase(old_uuid_hex, '-'); + const String probe = "p/roots/victim/clickhouse_access_check_" + old_uuid; { - return clock->nowFn()(); - }, - [clock](uint64_t ms) + auto victim = Pool::open(backend, victimConfig(hexToU128(old_uuid_hex))); + if (leftover == ServerRootLeftover::CatalogRows) + makeTableWithRefs(*victim, "victim/db/t1", 1, 0); + } + const auto report = decommissionPoolMember(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + ASSERT_TRUE(report.member_retired); + ASSERT_EQ(report.namespaces_pending_gc, leftover == ServerRootLeftover::CatalogRows ? 1u : 0u); + + String part; + std::function clear; + switch (leftover) { - clock->sleepFn()(ms); - }); - EXPECT_FALSE(first.warnings.empty()); - EXPECT_FALSE(first.slot_removed); - EXPECT_TRUE((*raw_op).head("p/gc/server-roots/victim/mount", Retry::once()).has_value()) - << "slot kept -- resume anchor"; + case ServerRootLeftover::CatalogRows: + part = "catalog rows of the old life"; + clear = [&] { gcUntilVictimSubtreeEmpty(backend); }; + break; + case ServerRootLeftover::Manifests: + ASSERT_TRUE(std::holds_alternative((*op).create("p/cas/manifests/victim/m", "m", Retry::once()))); + part = "manifests under cas/manifests/victim/"; + clear = [&] { ASSERT_EQ((*op).removeCurrent("p/cas/manifests/victim/m", Retry::once()), Removal::Removed); }; + break; + case ServerRootLeftover::RootsObjects: + ASSERT_TRUE(std::holds_alternative((*op).create(probe, "test", Retry::once()))); + ASSERT_TRUE(std::holds_alternative((*op).create("p/roots/victim/backup.zip", "zip", Retry::once()))); + part = "an object under roots/victim/"; + clear = [&] { ASSERT_EQ((*op).removeCurrent("p/roots/victim/backup.zip", Retry::once()), Removal::Removed); }; + break; + case ServerRootLeftover::None: + FAIL(); + } + const std::vector before = control_objects(*backend); - failing->disarm(); - const auto second = decommissionPoolMember( - failing, PoolConfig{.pool_prefix = "p", .server_root_id = "a2"}, "victim"); - EXPECT_FALSE(second.warnings.empty()); - EXPECT_FALSE(second.slot_removed); - EXPECT_EQ(second.namespaces_already_removed, 1u); - EXPECT_EQ(second.mountpoint_objects_removed, 1u); + try + { + Pool::open(backend, victimConfig(UInt128(77))); + ADD_FAILURE() << "the rejoin was not refused"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ABORTED) << e.message(); + EXPECT_NE(e.message().find(part), String::npos) << e.message(); + if (leftover == ServerRootLeftover::CatalogRows) + EXPECT_NE(e.message().find("victim/db/t1 (removing)"), String::npos) << "the refusal names the row"; + if (leftover != ServerRootLeftover::RootsObjects) + EXPECT_NE(e.message().find("on the GC leader"), String::npos) << e.message(); + } - drainCompletedNamespaceRemovals(inner); - const auto third = decommissionPoolMember( - failing, PoolConfig{.pool_prefix = "p", .server_root_id = "a3"}, "victim"); - EXPECT_TRUE(third.warnings.empty()); - EXPECT_TRUE(third.slot_removed); + EXPECT_EQ(control_objects(*backend), before); + if (leftover == ServerRootLeftover::RootsObjects) + EXPECT_FALSE((*op).head(probe, Retry::once()).has_value()) << "the old life's access probe is deleted first"; + clear(); + EXPECT_NO_THROW(Pool::open(backend, victimConfig(UInt128(77)))); + } } -/// Task 4 fail-close, manifest-debris variant (review follow-up: the plan's own example only exercises -/// a roots-phase failure). A per-key `deleteExact` throw inside the manifest-debris drain must ALSO -/// keep the slot: `report.slot_removed == false`, the mount object survives, and once the injected -/// failure is cleared a re-run drains the leftover debris and removes the slot. -TEST(CASDecommission, ManifestDebrisFailureKeepsSlotThenResumes) +/// A rerun over a verified retirement died holding the slot. The rejoin waits until GC fences the lapsed lease. +TEST(CASRejoin, RejoinsOnceGcFencesAKilledRerunsLease) { - auto backend = std::make_shared(); - String debris_key; + for (const UInt128 uuid : {victim_uuid, UInt128(77)}) { - auto victim = openVictim(backend); - makeTableWithRefs(*victim, "victim/db/t1", 1, 0); - const ManifestId debris_id = seedOrphanManifestBody(*victim, "victim/db/t1"); - debris_key = victim->layout().manifestKey(debris_id); + SCOPED_TRACE(u128ToHex(uuid)); + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + const Layout layout("p"); + OperationForTest op(*backend); + const uint64_t rerun_epoch = allocateWriterEpoch(*op, layout, "victim", [] { return RefCatalog{}; }); + rewriteSlot(*backend, layout, [&](MountLease & body) + { + body.writer_epoch = rerun_epoch; + body.min_active_build_sequence = 0; + }); + tombstoneOwnerForTest(*backend, layout.ownerKey("victim")); + ASSERT_TRUE(readOwner(*backend, layout, "victim").retirement_verified); + + expectThrowsCodeWithMessage(ErrorCodes::ABORTED, "still running", [&] { Pool::open(backend, victimConfig(uuid)); }); + + MountObservationMap observations; + std::set warned; + for (const uint64_t mono_ms : {uint64_t{0}, uint64_t{1'000'000'000}}) + computeHeartbeatFloor(*op, layout, /*now_ms=*/0, [mono_ms] { return mono_ms; }, /*observer_lease_ms=*/30'000, + /*cadence_ms=*/1'000, observations, warned); + ASSERT_TRUE(readSlot(*backend, layout, "victim").gc_fenced); + + auto rejoined = Pool::open(backend, victimConfig(uuid)); + EXPECT_EQ(readOwner(*backend, layout, "victim").server_uuid, uuid); + EXPECT_TRUE(readSlot(*backend, layout, "victim").isHeldBy(uuid, rejoined->writerEpoch())); } - backend->failWithThrow(debris_key); - backend->latch(); +} - /// The engine reissues an unresolved delete until its own retry window closes, measured on this - /// clock, so the latched fault reaches a genuine give-up with no real time passing. - auto clock = std::make_shared(); - const auto first = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "a1"}, "victim", - /*sink=*/{}, /*request_gc_round=*/{}, - [clock] +TEST(CASRejoin, RefusesSlotRetiredBy2664) +{ + const Layout layout("p"); + { + SCOPED_TRACE("26.6.4 end state: tombstone without life or mark, no epoch, no mount"); + auto backend = std::make_shared(); + { auto other = Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "other"}); } + OperationForTest op(*backend); + const String tombstone = encodeOwner(OwnerObject{.server_uuid = UInt128{}, .retired_at_ms = 1}); + ASSERT_TRUE(std::holds_alternative((*op).create(layout.ownerKey("victim"), tombstone, Retry::once()))); + try { - return clock->nowFn()(); - }, - [clock](uint64_t ms) + Pool::open(backend, victimConfig()); + ADD_FAILURE() << "the start was not refused"; + } + catch (const DB::Exception & e) { - clock->sleepFn()(ms); - }); - EXPECT_FALSE(first.warnings.empty()); - EXPECT_FALSE(first.slot_removed); - EXPECT_EQ(first.manifest_debris_removed, 0u); - OperationForTest raw_op(*backend); - EXPECT_TRUE((*raw_op).head("p/gc/server-roots/victim/mount", Retry::once()).has_value()) - << "slot kept -- resume anchor"; - EXPECT_TRUE((*raw_op).head(debris_key, Retry::once()).has_value()) - << "the failing object is left behind (untouched) so a re-run can retry it"; - - /// COVERAGE LOST HERE, DELIBERATELY NAMED. Before the §6 premise, clearing the injected failure let - /// a re-run drain the debris and retire the slot, which is what proved the per-key fail-close path - /// RESUMES rather than merely refuses. Under the premise the sweep never reaches `deleteExact` for - /// this body at all (single-epoch pool -- see the note on the helpers above), so disarming changes - /// nothing and the resume half of this test is no longer expressible. What survives is the half that - /// still has a mechanism: the slot stays kept across the re-run, and the object stays untouched. - /// The resume assertion comes back with the drain's reclaim path (registers R2/R3, Stage B). - backend->disarm(); - const auto second = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "a2"}, "victim"); - EXPECT_EQ(second.namespaces_already_removed, 1u); - EXPECT_EQ(second.manifest_debris_removed, 0u); - EXPECT_FALSE(second.slot_removed); - EXPECT_TRUE((*raw_op).head(debris_key, Retry::once()).has_value()); - EXPECT_TRUE((*raw_op).head("p/gc/server-roots/victim/mount", Retry::once()).has_value()) - << "the slot is still the resume anchor -- nothing was retired against unreclaimed debris"; -} - -/// CI run 7 (PR #2300): `drain_now_fn` fakes the drain's own request clock, but the mount lease's -/// farewell deadline is bound to `PoolConfig::boot_ms_fn`, a distinct clock that -- before this test's -/// fix -- stayed on the real boot clock regardless. On a freshly booted CI host (real boot time below -/// `FakeClock`'s starting instant) the two clocks disagreed enough that the farewell's own bound looked -/// already-expired, so `admin.reset()` released nothing and the slot was never retired -- passing locally -/// only because a long-lived dev box's real uptime dwarfs the fake clock. Pin the fake clock far beyond -/// ANY real host's boot time (rather than relying on the host actually being fresh) so the mismatch -- -/// and the fix -- are exercised deterministically on every machine. -TEST(CASDecommission, DrainClockUnifiesWithTheFarewellBootClockRegardlessOfHostUptime) + EXPECT_EQ(e.code(), ErrorCodes::CORRUPTED_DATA) << e.message(); + EXPECT_NE(e.message().find("retired by an older version"), String::npos) << e.message(); + EXPECT_EQ(e.message().find("not finished"), String::npos) << "a rerun refuses the absent counter: " << e.message(); + } + EXPECT_EQ((*op).read(layout.ownerKey("victim"), Retry::once())->bytes, tombstone); + EXPECT_FALSE((*op).head(layout.epochKey("victim"), Retry::once()).has_value()); + EXPECT_FALSE((*op).head(layout.mountKey("victim"), Retry::once()).has_value()); + } + { + SCOPED_TRACE("verified retirement whose epoch counter was deleted by hand"); + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + OperationForTest op(*backend); + ASSERT_EQ((*op).removeCurrent(layout.epochKey("victim"), Retry::once()), Removal::Removed); + expectThrowsCodeWithMessage(ErrorCodes::CORRUPTED_DATA, "cannot rejoin", [&] { Pool::open(backend, victimConfig()); }); + EXPECT_FALSE((*op).head(layout.epochKey("victim"), Retry::once()).has_value()); + EXPECT_TRUE(readOwner(*backend, layout, "victim").retired_at_ms.has_value()); + } + { + SCOPED_TRACE("verified retirement whose mount slot was deleted by hand"); + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + OperationForTest op(*backend); + ASSERT_EQ((*op).removeCurrent(layout.mountKey("victim"), Retry::once()), Removal::Removed); + expectThrowsCodeWithMessage(ErrorCodes::CORRUPTED_DATA, "cannot rejoin", [&] { Pool::open(backend, victimConfig()); }); + EXPECT_FALSE((*op).head(layout.mountKey("victim"), Retry::once()).has_value()); + EXPECT_TRUE(readOwner(*backend, layout, "victim").retired_at_ms.has_value()); + } +} + +/// Starter 88 rewrote the slot and stalled before its owner write; starter 77 then writes its owner and +/// takes the loser's terminal body. A later start of 88 is refused and writes nothing. +TEST(CASRejoin, ConcurrentStartersWithDifferentUuidsOneWins) { auto backend = std::make_shared(); - { auto victim = openVictim(backend); } /// identity only -- no namespace, so retirement runs straight - /// to the farewell instead of stopping on an unrelated warning - - auto clock = std::make_shared(); - clock->now = 1'000'000'000'000'000ULL; /// dwarfs any real CLOCK_BOOTTIME on any host - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "a1"}, "victim", - /*sink=*/{}, /*request_gc_round=*/{}, - [clock] - { - return clock->nowFn()(); - }, - [clock](uint64_t ms) - { - clock->sleepFn()(ms); - }); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + const Layout layout("p"); + bool loser_wrote_mount = false; + backend->onBeforeWrite(layout.ownerKey("victim"), [&] + { + if (std::exchange(loser_wrote_mount, true)) + return; + rewriteSlot(*backend, layout, [](MountLease & body) { body.server_uuid = UInt128(88); }); + }); - EXPECT_TRUE(report.warnings.empty()); - EXPECT_TRUE(report.slot_removed); - OperationForTest raw_op(*backend); - EXPECT_FALSE((*raw_op).head("p/gc/server-roots/victim/mount", Retry::once()).has_value()) - << "one clock for both the request engine and the mount lease -- the farewell must commit and the " - "slot must retire in a single call, with no leftover mount to resume against"; -} - -/// Review round 9r2: folding `drain_now_fn` into `config.boot_ms_fn` alone left `Pool::openForDecommission`'s -/// own mount/farewell/GC planes -- constructed and used DURING opening, before `decommissionPoolMember` -/// gets a chance to call `setCasRequestNowFnForTest`/`setCasRetrySleepForTest` on the already-open `Pool` -/// -- retrying on a clock that only a fake SLEEP advances, while those planes still slept for REAL between -/// attempts. A transient failure during opening (the owner-object read on the open plane) would then never -/// see its own `Retry::standard()` deadline elapse, because the bound is measured against a clock frozen -/// at the value it had when the retry loop started: nothing calls the fake clock's `sleepFn` from a path -/// that still sleeps for real. `PoolConfig::retry_sleep_fn` closes this by installing the matching fake -/// sleep on those same planes AT CONSTRUCTION, together with `boot_ms_fn`. -/// -/// Failing-first without risking an actual hang: this fixture flakes the owner read 5 times, so a correct -/// fix paces exactly 5 retries on the fake clock and returns in well under a second of real time; the -/// pre-fix code either never returns (the bound never elapses) or, if it did return, would show an empty -/// `clock.sleeps` (nothing ever called the fake sleep) and real wall time consumed by 5 real backoffs. -TEST(CASDecommission, OpeningRetriesPaceOnTheSameFakeClockAndSleepAsTheDrain) -{ - auto backend = std::make_shared(); - { auto victim = openVictim(backend); } /// identity only + auto winner = Pool::open(backend, victimConfig(UInt128(77))); + + ASSERT_TRUE(loser_wrote_mount); + EXPECT_EQ(readOwner(*backend, layout, "victim").server_uuid, UInt128(77)); + EXPECT_EQ(readSlot(*backend, layout, "victim").server_uuid, UInt128(77)); + expectThrowsCodeWithMessage(ErrorCodes::CORRUPTED_DATA, "different server", [&] { Pool::open(backend, victimConfig(UInt128(88))); }); + EXPECT_EQ(readSlot(*backend, layout, "victim").writer_epoch, winner->writerEpoch()); +} +/// Starter 88's owner write loses its `If-Match` to starter 77's owner write: it decides again from a fresh +/// owner read, sees the foreign owner and refuses. It has already rewritten the slot, as every new uuid does +/// before its owner write, and leaves the owner alone. +TEST(CASRejoin, LostOwnerWriteDecidesAgainAndRefuses) +{ + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); const Layout layout("p"); - backend->failReadNTimes(layout.ownerKey("victim"), /*times=*/5); + const OwnerObject retired = readOwner(*backend, layout, "victim"); + const MountLease slot_before = readSlot(*backend, layout, "victim"); + const uint64_t slot_seq = slot_before.seq; + bool winner_wrote = false; + String winner_body; + backend->onBeforeWrite(layout.ownerKey("victim"), [&] + { + if (std::exchange(winner_wrote, true)) + return; + OperationForTest op(*backend); + const auto current = (*op).read(layout.ownerKey("victim"), Retry::once()); + winner_body = encodeOwner(OwnerObject{.server_uuid = UInt128(77), .retired_at_ms = std::nullopt, .owner_life = retired.owner_life + 1}); + ASSERT_TRUE(std::holds_alternative((*op).replace(layout.ownerKey("victim"), winner_body, current->etag, Retry::once()))); + }); - auto clock = std::make_shared(); - clock->now = 1'000'000'000'000'000ULL; + expectThrowsCodeWithMessage(ErrorCodes::CORRUPTED_DATA, "different server", [&] { Pool::open(backend, victimConfig(UInt128(88))); }); - const auto started = std::chrono::steady_clock::now(); - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "a1"}, "victim", - /*sink=*/{}, /*request_gc_round=*/{}, - [clock] - { - return clock->nowFn()(); - }, - [clock](uint64_t ms) - { - clock->sleepFn()(ms); - }); - const auto wall_elapsed = std::chrono::steady_clock::now() - started; + ASSERT_TRUE(winner_wrote); + OperationForTest op(*backend); + EXPECT_EQ((*op).read(layout.ownerKey("victim"), Retry::once())->bytes, winner_body) << "the loser left the owner alone"; + const MountLease slot = readSlot(*backend, layout, "victim"); + EXPECT_EQ(slot.seq, slot_seq + 1); + EXPECT_EQ(slot.server_uuid, UInt128(88)); + EXPECT_NE(slot.write_attempt_id, slot_before.write_attempt_id); + EXPECT_EQ(slot.pid, static_cast(::getpid())); +} - EXPECT_TRUE(report.warnings.empty()); - EXPECT_TRUE(report.slot_removed); - EXPECT_FALSE(clock->sleeps.empty()) - << "the opening retries must have been paced on the injected clock, not a real sleep"; - EXPECT_LT(wall_elapsed, std::chrono::seconds(5)) - << "paced on the fake clock, five retries during opening should cost no real wall time at all"; -} - -/// Task 5 (Task-1 carry-forward, escalated by review): preserve recovery from the legacy partial -/// hand-cleanup shape where owner and epoch are absent but the mount lease remains. Triage #9 changed -/// new retirements to delete `mountKey`/`epochKey` and tombstone `ownerKey`, so the current tail no -/// longer creates this shape, but `openForDecommission`'s owner-anchor-absent + -/// mount-lease-present fallback ("partial hand-cleanup: adopt from the lease", `CasPool.cpp`) remains -/// compatibility-critical for slots left by older binaries or manual repair. -/// -/// `claimOwnerOrThrow` (`CasServerRoot.cpp`) gates the owner-absent path a SECOND, stricter way: the -/// same catalog cut must name no `Creating`, `Live` or `Removing` namespace under this canonical root, -/// and the name-bearing `cas/manifests//` and `roots//` families must be empty. Opaque -/// stream/state debris cannot be attributed to a server root and is deliberately inert. This test -/// therefore uses a victim with NO namespaces at all: identity persisted -/// (mount/owner/epoch exist from a real graceful close), data subtree genuinely empty -- the exact -/// precondition the fallback is designed for. Simulate the crash directly: claim the slot once (exactly -/// `decommissionPoolMember`'s own first step), let it close gracefully (the mount-lease renewer's -/// farewell stamp, same as a real `admin.reset()`), then manually strike `epochKey`+`ownerKey`, leaving -/// `mountKey`. A `decommissionPoolMember` re-run must resolve identity via the mount-lease fallback and -/// finish retiring the slot; a further re-run then sees the tombstone and refuses to resume it. -TEST(CASDecommission, MidRetirementCrashResumesViaMountLeaseFallback) +/// The old uuid rejoins after a new uuid replaced the mount body, and the new uuid writes the owner first. The old +/// uuid writes no body of its own: under the new owner it would be a farewell a 26.6.4 starter of the old uuid, +/// prechecked before the decommission, could still reclaim. +TEST(CASRejoin, SameUuidLeavesTheMountBodyAlone) +{ + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + const Layout layout("p"); + const OwnerObject retired = readOwner(*backend, layout, "victim"); + rewriteSlot(*backend, layout, [](MountLease & body) { body.server_uuid = UInt128(77); }); + const MountLease replaced = readSlot(*backend, layout, "victim"); + std::atomic fired = false; + backend->onBeforeWrite(layout.ownerKey("victim"), [&] + { + if (fired.exchange(true)) + return; + OperationForTest op(*backend); + const auto current = (*op).read(layout.ownerKey("victim"), Retry::once()); + const String body = encodeOwner( + OwnerObject{.server_uuid = UInt128(77), .retired_at_ms = std::nullopt, .owner_life = retired.owner_life + 1}); + ASSERT_TRUE(std::holds_alternative( + (*op).replace(layout.ownerKey("victim"), body, current->etag, Retry::once()))); + }); + + expectThrowsCodeWithMessage(ErrorCodes::CORRUPTED_DATA, "different server", [&] { Pool::open(backend, victimConfig()); }); + + backend->onBeforeWrite(layout.ownerKey("victim"), {}); + ASSERT_TRUE(fired); + const MountLease slot = readSlot(*backend, layout, "victim"); + EXPECT_EQ(slot.server_uuid, UInt128(77)) << "the old uuid wrote no body under the new owner"; + EXPECT_EQ(slot.seq, replaced.seq); +} + +/// Every rejoin write loses its `If-Match`: the start gives up after a bounded number of decisions. +TEST(CASRejoin, OwnerChangingOnEveryDecisionIsBounded) { auto backend = std::make_shared(); - { auto victim = openVictim(backend); } /// identity only -- no namespace, so the subtree stays empty + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + const Layout layout("p"); + bool in_hook = false; + size_t fires = 0; + backend->onBeforeWrite(layout.ownerKey("victim"), [&] + { + if (std::exchange(in_hook, true)) + return; + ++fires; + OperationForTest op(*backend); + const auto current = (*op).read(layout.ownerKey("victim"), Retry::once()); + ASSERT_TRUE(std::holds_alternative((*op).replace(layout.ownerKey("victim"), current->bytes, current->etag, Retry::once()))); + in_hook = false; + }); + + expectThrowsCodeWithMessage(ErrorCodes::ABORTED, "kept changing", [&] { Pool::open(backend, victimConfig(UInt128(77))); }); + + EXPECT_EQ(fires, 4u); + EXPECT_TRUE(readOwner(*backend, layout, "victim").retired_at_ms.has_value()); +} + +/// A start of the old life allocated its epoch before the decommission and claims only after a rejoin. +TEST(CASRejoin, DelayedLowerEpochClaimAfterRejoinIsRefused) +{ + for (const UInt128 rejoiner : {victim_uuid, UInt128(77)}) + { + SCOPED_TRACE(rejoiner == victim_uuid ? "same uuid" : "new uuid: a foreign farewell"); + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + const Layout layout("p"); + uint64_t delayed_epoch = 0; + { + OperationForTest op(*backend); + delayed_epoch = allocateWriterEpoch(*op, layout, "victim", [] { return RefCatalog{}; }); + } + retireVictim(backend, "admin"); + { auto rejoined = Pool::open(backend, victimConfig(rejoiner)); } + OperationForTest op(*backend); + const auto before = (*op).read(layout.mountKey("victim"), Retry::once()); + const OwnerExpectation old_life{.server_uuid = victim_uuid, .owner_life = 1, .retired = RetiredOwnerPolicy::Refuse}; + if (rejoiner == victim_uuid) + expectThrowsCode(ErrorCodes::ABORTED, [&] { claimMount(*op, layout, "victim", victim_uuid, delayed_epoch, 1, 100, {}, {}, {}, old_life); }); + else + EXPECT_EQ(claimMount(*op, layout, "victim", victim_uuid, delayed_epoch, 1, 100, {}, {}, {}, old_life).kind, MountClaimResult::ForeignOwner); + EXPECT_EQ((*op).read(layout.mountKey("victim"), Retry::once())->etag, before->etag); + } +} +/// The victim restarts while the admin holds the slot; it reclaims the farewell, sees the tombstone and +/// fences itself. A completed rerun later, the same victim rejoins on its next start. +TEST(CASRejoin, VictimRefusedAfterFarewellRejoinsOnItsNextStart) +{ + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } const Layout layout("p"); - /// Claim the slot once, exactly as `decommissionPoolMember`'s own first step would -- this (re)writes - /// fresh epoch/owner/mount control objects. Closing gracefully (scope exit) stamps the mount lease's - /// farewell, matching what a real slot retirement's `admin.reset()` does right before its delete loop. + auto admin = Pool::openForDecommission(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); + auto fake_boot = std::make_shared>(0); + bool decommissioned = false; + PoolConfig waiting = victimConfig(); + waiting.boot_ms_fn = [fake_boot] { return fake_boot->load(); }; + waiting.wait_sleep_fn = [&, fake_boot](uint64_t ms) + { + *fake_boot += ms; + if (std::exchange(decommissioned, true)) + return; + tombstoneOwnerForTest(*backend, layout.ownerKey("victim")); + admin.reset(); + }; + expectThrowsCodeWithMessage(ErrorCodes::CORRUPTED_DATA, "decommissioned", [&] { Pool::open(backend, waiting); }); + ASSERT_TRUE(readSlot(*backend, layout, "victim").gc_fenced); + + retireVictim(backend, "admin2"); + auto rejoined = Pool::open(backend, victimConfig()); + EXPECT_FALSE(readOwner(*backend, layout, "victim").retired_at_ms.has_value()); + EXPECT_EQ(readSlot(*backend, layout, "victim").writer_epoch, rejoined->writerEpoch()); +} + +TEST(CASRejoin, RefusesWhileAnUnfinishedDecommissionLeftData) +{ + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } { - auto admin = Pool::openForDecommission(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "chk"}, "victim"); + OperationForTest op(*backend); + ASSERT_TRUE(std::holds_alternative((*op).create("p/roots/victim/kept", "x", Retry::once()))); } + backend->failWithTokenMismatch("p/roots/victim/kept"); + ASSERT_FALSE(decommissionPoolMember(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim").member_retired); + const Layout layout("p"); + const OwnerObject tombstone = readOwner(*backend, layout, "victim"); + ASSERT_TRUE(tombstone.retired_at_ms.has_value()); + ASSERT_FALSE(tombstone.retirement_verified); - /// Manually strike epoch + owner, leaving the mount -- the legacy partial hand-cleanup shape. - OperationForTest raw_op(*backend); - for (const String & key : {layout.epochKey("victim"), layout.ownerKey("victim")}) + for (const UInt128 uuid : {victim_uuid, UInt128(77)}) { - const auto head = (*raw_op).head(key, Retry::once()); - ASSERT_TRUE(head.has_value()); - ASSERT_EQ((*raw_op).remove(key, head->etag, Retry::once()), Removal::Removed); + SCOPED_TRACE(u128ToHex(uuid)); + expectThrowsCodeWithMessage(ErrorCodes::CORRUPTED_DATA, "decommission is not finished", [&] { Pool::open(backend, victimConfig(uuid)); }); } - ASSERT_FALSE((*raw_op).head(layout.epochKey("victim"), Retry::once()).has_value()); - ASSERT_FALSE((*raw_op).head(layout.ownerKey("victim"), Retry::once()).has_value()); - ASSERT_TRUE((*raw_op).head(layout.mountKey("victim"), Retry::once()).has_value()) - << "the mount lease must survive -- it is the resume anchor the fallback reads"; + EXPECT_EQ(readOwner(*backend, layout, "victim").owner_life, tombstone.owner_life); + + backend->disarm(); + retireVictim(backend, "admin2"); + EXPECT_NO_THROW(Pool::open(backend, victimConfig())); +} + +/// A losing racing rejoiner of uuid 88 left its terminal body; the old uuid's own restart still rejoins. +TEST(CASRejoin, SameUuidRejoinsOverAForeignTerminalBody) +{ + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + const Layout layout("p"); + rewriteSlot(*backend, layout, [](MountLease & body) { body.server_uuid = UInt128(88); }); - const auto report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "a2"}, "victim"); + auto rejoined = Pool::open(backend, victimConfig()); - EXPECT_TRUE(report.warnings.empty()); - EXPECT_EQ(report.namespaces_removed, 0u); - EXPECT_TRUE(report.slot_removed); - EXPECT_FALSE((*raw_op).head(layout.epochKey("victim"), Retry::once()).has_value()); - const auto owner = (*raw_op).read(layout.ownerKey("victim"), Retry::once()); - ASSERT_TRUE(owner.has_value()); - EXPECT_TRUE(decodeOwner(owner->bytes).retired_at_ms.has_value()); - EXPECT_FALSE((*raw_op).head(layout.mountKey("victim"), Retry::once()).has_value()); + const MountLease slot = readSlot(*backend, layout, "victim"); + EXPECT_EQ(slot.server_uuid, victim_uuid); + EXPECT_EQ(slot.writer_epoch, rejoined->writerEpoch()); +} + +/// Only a start rejoins. A process that was fenced and finds its id retired keeps refusing to remount. +TEST(CASRejoin, RemountDoesNotRejoin) +{ + auto backend = std::make_shared(); + auto store = Pool::open(backend, victimConfig()); + const Layout layout("p"); + tombstoneOwnerForTest(*backend, layout.ownerKey("victim"), /*verified=*/true); + fenceOutMountForTest(*backend, layout.mountKey("victim")); + expectThrowsCode(ErrorCodes::ABORTED, [&] { store->renewWatermarkOnce(); }); + const OwnerObject before = readOwner(*backend, layout, "victim"); + + EXPECT_FALSE(store->tryRemountOnce()); + EXPECT_EQ(store->lifecycle(), PoolLifecycle::IdentityLost); + + const OwnerObject after = readOwner(*backend, layout, "victim"); + EXPECT_EQ(after.owner_life, before.owner_life); + EXPECT_TRUE(after.retired_at_ms.has_value()); +} - expectThrowsCode(ErrorCodes::CORRUPTED_DATA, [&] +/// The member was fenced while still running, decommissioned, and its id rejoined under a new uuid. The old +/// process stops for good instead of retrying. +TEST(CASRejoin, OldProcessStopsForGoodAfterANewUuidRejoin) +{ + auto backend = std::make_shared(); + auto old_process = Pool::open(backend, victimConfig()); + const Layout layout("p"); + fenceOutMountForTest(*backend, layout.mountKey("victim")); + expectThrowsCode(ErrorCodes::ABORTED, [&] { old_process->renewWatermarkOnce(); }); + retireVictim(backend, "admin"); + auto rejoined = Pool::open(backend, victimConfig(UInt128(77))); + const MountLease slot_before = readSlot(*backend, layout, "victim"); + + EXPECT_FALSE(old_process->tryRemountOnce()); + EXPECT_EQ(old_process->lifecycle(), PoolLifecycle::IdentityLost); + + const MountLease slot_after = readSlot(*backend, layout, "victim"); + EXPECT_EQ(slot_after.server_uuid, UInt128(77)); + EXPECT_EQ(slot_after.writer_epoch, slot_before.writer_epoch) << "the old process wrote nothing"; +} + + +/// A 26.6.4 starter of the old uuid read the farewell before the rejoin and reclaims it by `If-Match` with +/// no owner recheck; it fires just before the rejoin's owner write. +TEST(CASRejoin, NewUuidReplacesTheMountBeforeTheOwner) +{ + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + const Layout layout("p"); + OperationForTest raw_op(*backend); + const auto old_farewell = (*raw_op).read(layout.mountKey("victim"), Retry::once()); + ASSERT_TRUE(old_farewell.has_value()); + std::optional old_claim; + backend->onBeforeWrite(layout.ownerKey("victim"), [&] { - decommissionPoolMember(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "a3"}, "victim"); + if (old_claim) + return; + MountLease body = decodeMountLease(old_farewell->bytes); + body.writer_epoch += 1; + body.min_active_build_sequence = 0; + body.seq += 1; + OperationForTest op(*backend); + old_claim = (*op).replace(layout.mountKey("victim"), encodeMountLease(body), old_farewell->etag, Retry::once()); }); + + auto rejoined = Pool::open(backend, victimConfig(UInt128(77))); + + ASSERT_TRUE(old_claim.has_value()); + EXPECT_TRUE(std::holds_alternative(*old_claim)) << "the rejoin rewrote the mount body first"; + EXPECT_EQ(readSlot(*backend, layout, "victim").server_uuid, UInt128(77)); +} + +/// A decommission rerun claims the slot between the rejoin's reads and its first write: the rejoin's +/// `If-Match` fails, the next decision sees the admin's live body and refuses; the owner stays retired. +TEST(CASRejoin, DecommissionRerunDuringARejoinWins) +{ + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig()); } + retireVictim(backend, "admin"); + const Layout layout("p"); + const OwnerObject before = readOwner(*backend, layout, "victim"); + std::atomic fired = false; + PoolPtr rerun_admin; + const auto admin_claims = [&] + { + if (fired.exchange(true)) + return; + rerun_admin = Pool::openForDecommission(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin2"}, "victim"); + }; + backend->onBeforeWrite(layout.mountKey("victim"), admin_claims); + backend->onBeforeWrite(layout.ownerKey("victim"), admin_claims); + + expectThrowsCodeWithMessage(ErrorCodes::ABORTED, "still running", [&] { Pool::open(backend, victimConfig(UInt128(77))); }); + + backend->onBeforeWrite(layout.mountKey("victim"), {}); + backend->onBeforeWrite(layout.ownerKey("victim"), {}); + ASSERT_TRUE(rerun_admin != nullptr); + const OwnerObject after = readOwner(*backend, layout, "victim"); + EXPECT_EQ(after.server_uuid, before.server_uuid); + EXPECT_EQ(after.owner_life, before.owner_life); + EXPECT_TRUE(after.retirement_verified); + EXPECT_EQ(readSlot(*backend, layout, "victim").writer_epoch, rerun_admin->writerEpoch()); +} + +/// A writable open without a server uuid (uuid 0) stays refused over a retired owner, whoever owned it. +TEST(CASRejoin, ZeroUuidDoesNotRejoin) +{ + for (const UInt128 old_uuid : {UInt128(77), UInt128{}}) + { + SCOPED_TRACE(u128ToHex(old_uuid)); + auto backend = std::make_shared(); + { auto victim = Pool::open(backend, victimConfig(old_uuid)); } + retireVictim(backend, "admin"); + const Layout layout("p"); + const OwnerObject before = readOwner(*backend, layout, "victim"); + const MountLease slot_before = readSlot(*backend, layout, "victim"); + + expectThrowsCodeWithMessage( + ErrorCodes::CORRUPTED_DATA, "no server uuid", + [&] { Pool::open(backend, victimConfig(UInt128{})); }); + + const OwnerObject after = readOwner(*backend, layout, "victim"); + EXPECT_EQ(after.owner_life, before.owner_life); + EXPECT_TRUE(after.retired_at_ms.has_value()); + EXPECT_EQ(readSlot(*backend, layout, "victim").seq, slot_before.seq); + } } diff --git a/src/Disks/tests/gtest_cas_decommission_catalog_duties.cpp b/src/Disks/tests/gtest_cas_decommission_catalog_duties.cpp index 34cb8b7fad077..09400c1ed89fc 100644 --- a/src/Disks/tests/gtest_cas_decommission_catalog_duties.cpp +++ b/src/Disks/tests/gtest_cas_decommission_catalog_duties.cpp @@ -54,6 +54,15 @@ bool slotObjectExists(Backend & backend, const String & leaf) return (*op).head("p/gc/server-roots/victim/" + leaf, Retry::standard()).has_value(); } +OwnerObject readOwnerDuty(Backend & backend) +{ + DB::Cas::tests::OperationForTest op(backend); + const auto current = (*op).read("p/gc/server-roots/victim/owner", Retry::standard()); + if (!current) + throw Exception(ErrorCodes::CORRUPTED_DATA, "fixture owner is absent"); + return decodeOwner(current->bytes); +} + class AddVictimEntryDuringRootDrainBackend final : public InMemoryBackend { public: @@ -87,60 +96,7 @@ class AddVictimEntryDuringRootDrainBackend final : public InMemoryBackend bool added = false; }; -/// Admits the late catalog entry between the retirement tail's two exact catalog reads -/// (`retirement_catalog_cut`, then `fresh_retirement_catalog`), never before. The mountpoint drain's -/// `list("p/roots/victim/", ...)` is the last LIST call in `decommissionPoolMember` before either -/// read, so it orders the two `read("p/cas/ref_catalog")` calls that follow it: the first is -/// `retirement_catalog_cut`, the second is `fresh_retirement_catalog`. Mutating on the second call -/// makes that read observe a catalog the first read did not. -class MutateCatalogBetweenRetirementReadsBackend final : public InMemoryBackend -{ -public: - /// Unhide the legacy `list` overloads the primitive override below would otherwise hide. - using Backend::list; - void arm() { armed = true; } - bool fired() const { return added; } - - /// Both hooks sit on the PRIMITIVES, which every legacy forwarder reaches too, so the ordering - /// they observe is the physical request order whichever surface issued each request. - RawListPage list(const String & prefix, const String & cursor, size_t limit, TransportAccess & access) override - { - RawListPage page = InMemoryBackend::list(prefix, cursor, limit, access); - if (armed && !past_mountpoint_drain && prefix == "p/roots/victim/" && cursor.empty()) - past_mountpoint_drain = true; - return page; - } - - std::optional read(const String & key, TransportAccess & access) override - { - if (armed && past_mountpoint_drain && !added && key == "p/cas/ref_catalog") - { - if (!seen_retirement_catalog_cut) - seen_retirement_catalog_cut = true; - else - { - added = true; - CasRequests requests = DB::Cas::tests::openRequestsForTest(*this); - CasOperation op = requests.admit(); - CasRefCatalog::casAdmitEntry( - op, Layout("p"), 1, - CatalogEntry{ - .ns = RootNamespace("victim/db/late"), - .state = NsState::Live, - .incarnation = UInt128{707}}); - } - } - return InMemoryBackend::read(key, access); - } - -private: - bool armed = false; - bool past_mountpoint_drain = false; - bool seen_retirement_catalog_cut = false; - bool added = false; -}; - -TEST(CASDecommissionCatalogDuties, RemovingWithoutCheckpointIsCorruptionAndKeepsSlot) +TEST(CASDecommissionCatalogDuties, RemovingWithoutCheckpointIsCorruptionAndLeavesAnUnverifiedTombstone) { auto backend = std::make_shared(); CasRequests catalog_requests = DB::Cas::tests::openRequestsForTest(backend); @@ -166,11 +122,13 @@ TEST(CASDecommissionCatalogDuties, RemovingWithoutCheckpointIsCorruptionAndKeeps EXPECT_TRUE(slotObjectExists(*backend, "owner")); EXPECT_TRUE(slotObjectExists(*backend, "epoch")); EXPECT_TRUE(slotObjectExists(*backend, "mount")); + EXPECT_TRUE(readOwnerDuty(*backend).retired_at_ms.has_value()); + EXPECT_FALSE(readOwnerDuty(*backend).retirement_verified); EXPECT_EQ(catalogEntry(catalog_op, Layout("p"), RootNamespace("victim/db/missing_ckpt")).state, NsState::Removing); } -TEST(CASDecommissionCatalogDuties, RemovingWithCheckpointResumesTerminalAndKeepsSlotForGc) +TEST(CASDecommissionCatalogDuties, RemovingWithCheckpointResumesTerminalAndIsVerifiedWithTheRowPendingGc) { auto backend = std::make_shared(); CasRequests catalog_requests = DB::Cas::tests::openRequestsForTest(backend); @@ -193,8 +151,9 @@ TEST(CASDecommissionCatalogDuties, RemovingWithCheckpointResumesTerminalAndKeeps EXPECT_EQ(report.namespaces_already_removed, 1u); EXPECT_EQ(wake_requests.load(), 1u); - EXPECT_FALSE(report.slot_removed); - EXPECT_FALSE(report.warnings.empty()); + EXPECT_TRUE(report.member_retired); + EXPECT_TRUE(report.warnings.empty()); + EXPECT_EQ(report.namespaces_pending_gc, 1u); EXPECT_TRUE(slotObjectExists(*backend, "owner")); const ListPage stream = catalog_op.list(Layout("p").namespaceStreamPrefix(*life), "", 100, Retry::standard()); @@ -246,12 +205,14 @@ TEST(CASDecommissionCatalogDuties, PartialRemovalProgressStillWakesGcWhenLaterNa EXPECT_EQ(wake_requests.load(), 1u) << "progress already made for an earlier life must wake GC even when a later life fails closed"; EXPECT_TRUE(slotObjectExists(*backend, "owner")); + EXPECT_TRUE(readOwnerDuty(*backend).retired_at_ms.has_value()); + EXPECT_FALSE(readOwnerDuty(*backend).retirement_verified); const ListPage progressed_stream = catalog_op.list(Layout("p").namespaceStreamPrefix(*progressed_life), "", 100, Retry::standard()); ASSERT_EQ(progressed_stream.keys.size(), 1u); } -TEST(CASDecommissionCatalogDuties, VictimEntryAppearingBeforeTheOwnershipCutKeepsSlot) +TEST(CASDecommissionCatalogDuties, VictimEntryAppearingAfterTheCutBlocksTheMark) { auto backend = std::make_shared(); CasRequests catalog_requests = DB::Cas::tests::openRequestsForTest(backend); @@ -263,31 +224,11 @@ TEST(CASDecommissionCatalogDuties, VictimEntryAppearingBeforeTheOwnershipCutKeep backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); EXPECT_TRUE(backend->fired()); - EXPECT_FALSE(report.slot_removed); - ASSERT_FALSE(report.warnings.empty()); - EXPECT_NE(report.warnings.front().find("pool member decommission underway: 1 namespace(s)"), String::npos) - << report.warnings.front(); - EXPECT_TRUE(slotObjectExists(*backend, "owner")); - EXPECT_EQ(catalogEntry(catalog_op, Layout("p"), RootNamespace("victim/db/late")).state, NsState::Live); -} - -TEST(CASDecommissionCatalogDuties, CatalogTokenMovedBetweenOwnershipCutAndRetirementKeepsSlot) -{ - auto backend = std::make_shared(); - CasRequests catalog_requests = DB::Cas::tests::openRequestsForTest(backend); - CasOperation catalog_op = catalog_requests.admit(); - { auto victim = openVictim(backend); } - backend->arm(); - - const DecommissionReport report = decommissionPoolMember( - backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); - - EXPECT_TRUE(backend->fired()); - EXPECT_FALSE(report.slot_removed); + EXPECT_FALSE(report.member_retired); ASSERT_FALSE(report.warnings.empty()); - EXPECT_NE(report.warnings.front().find("catalog changed after the victim ownership check"), String::npos) - << report.warnings.front(); - EXPECT_TRUE(slotObjectExists(*backend, "owner")); + EXPECT_NE(report.warnings.front().find("appeared after"), String::npos) << report.warnings.front(); + EXPECT_TRUE(readOwnerDuty(*backend).retired_at_ms.has_value()); + EXPECT_FALSE(readOwnerDuty(*backend).retirement_verified); EXPECT_EQ(catalogEntry(catalog_op, Layout("p"), RootNamespace("victim/db/late")).state, NsState::Live); } @@ -325,7 +266,9 @@ TEST(CASDecommissionCatalogDuties, FoldedTerminalRemainsGcOwnedAndOnlyRequestsAn EXPECT_EQ(wake_requests.load(), 1u); EXPECT_EQ(report.namespaces_already_removed, 1u); - EXPECT_FALSE(report.slot_removed); + EXPECT_TRUE(report.member_retired); + EXPECT_TRUE(report.warnings.empty()); + EXPECT_EQ(report.namespaces_pending_gc, 1u); EXPECT_EQ(catalogEntry(catalog_op, Layout("p"), ns).state, NsState::Removing); std::vector stream_after; for (const ListedKey & key : catalog_op.list(Layout("p").namespaceStreamPrefix(*life), "", 100, Retry::standard()).keys) @@ -350,7 +293,7 @@ TEST(CASDecommissionCatalogDuties, OpaqueLifeDebrisWithoutCatalogOwnershipDoesNo backend, PoolConfig{.pool_prefix = "p", .server_root_id = "admin"}, "victim"); EXPECT_TRUE(report.warnings.empty()); - EXPECT_TRUE(report.slot_removed); + EXPECT_TRUE(report.member_retired); EXPECT_TRUE(catalog_op.head(debris_key, Retry::standard()).has_value()); } diff --git a/src/Disks/tests/gtest_cas_gc_ack_floor.cpp b/src/Disks/tests/gtest_cas_gc_ack_floor.cpp index f3bd6da9095a8..47befa6616924 100644 --- a/src/Disks/tests/gtest_cas_gc_ack_floor.cpp +++ b/src/Disks/tests/gtest_cas_gc_ack_floor.cpp @@ -940,8 +940,8 @@ void runExpiredMountFenceOutScenario(const PoolConfig & config) srid2_renewer.start(); ASSERT_FALSE(decodeMountLease(readObj(*backend, layout.mountKey(srid2))->bytes).gc_fenced); - // The fence-out threshold on the GC leader's OWN monotonic clock — mirrors the production formula - // in `Gc::runRegularRound` (ttl + 5% drift allowance + one round's worth of renewal slack). + // The fence-out threshold on the GC leader's OWN monotonic clock. The holder declares 100 ms, shorter + // than the observer's lease, so L is the observer's own: `L + L/20 + step`, step the renewal period. const uint64_t ttl_ms = static_cast(store->poolConfig().mount_lease_ttl_ms.count()); const uint64_t threshold_ms = ttl_ms + ttl_ms / 20 + static_cast(store->poolConfig().mount_renew_period.count()); @@ -1056,7 +1056,7 @@ void runSlowWalkSightingScenario(SightingCall sighting_call) const uint64_t ttl_ms = static_cast(store->poolConfig().mount_lease_ttl_ms.count()); const uint64_t threshold_ms = mountObservationThresholdMs( - ttl_ms, static_cast(store->poolConfig().mount_renew_period.count())); + ttl_ms, /*declared_lease_ms=*/0, static_cast(store->poolConfig().mount_renew_period.count())); /// Longer than `threshold - TTL`: a sighting credited from before this walk fences a holder /// whose last renewal still gives it authority. const uint64_t walk_ms = threshold_ms - ttl_ms + 1'000; @@ -1113,6 +1113,52 @@ void runSlowWalkSightingScenario(SightingCall sighting_call) EXPECT_TRUE(decodeMountLease(readObj(*backend, backend->watched_key)->bytes).gc_fenced); } +/// Defaults give the observer 30 s + 1.5 s + 10 s = 41.5 s; the holder's 60 s gives 73 s. +void runLongerDeclaredLeaseScenario(SightingCall early_call) +{ + uint64_t mono = 0; + auto backend = std::make_shared(); + auto store = Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "test"}); + const Layout & layout = store->layout(); + ASSERT_EQ(store->poolConfig().mount_lease_ttl_ms.count(), 30'000); + ASSERT_EQ(store->poolConfig().mount_renew_period.count(), 10'000); + Gc gc(store, kGc, [] { return uint64_t{1'000'000}; }, [&mono] { return mono; }); + + const RootNamespace ns{"00/aa@cas@"}; + const ManifestRef r = ref("srv-a:1", 1, 0xAA); + writeBlobBody(*backend, layout, DB::UInt128(7)); + writeManifestRaw(*backend, layout, ns, r, {blobEntryFor("a", DB::UInt128(7))}); + publishCommittedTransition(*backend, layout, ns, "tbl", std::nullopt, r); + + /// Claims once with a 60 s lease and never renews. + const String holder_srid = "long-lease-server"; + CasRequests renewer_requests = openRequestsForTest(backend); + MountLeaseRenewer holder(renewer_requests, renewer_requests, layout, holder_srid, DB::UInt128(0x3333), + /*writer_epoch=*/1, std::chrono::milliseconds(60'000), [] { return 1000u; }, [] { return 0u; }, {}, + std::chrono::milliseconds(0), [] { return 0u; }); + holder.start(); + const auto holder_fenced = [&] + { return decodeMountLease(readObj(*backend, layout.mountKey(holder_srid))->bytes).gc_fenced; }; + + ASSERT_EQ(gc.runRegularRound().fence_outs, 0u); + /// The store's own slot renews before each decision so only the holder can be fenced. + store->renewWatermarkOnce(); + mono = 41'500; + if (early_call == SightingCall::Round) + EXPECT_EQ(gc.runRegularRound().fence_outs, 0u); + else + ASSERT_TRUE(gc.rebuildBaseline(/*force*/ true).performed); + EXPECT_FALSE(holder_fenced()) << "fenced after the observer's own lease while the declared lease still ran"; + + store->renewWatermarkOnce(); + mono = 72'999; + EXPECT_EQ(gc.runRegularRound().fence_outs, 0u); + store->renewWatermarkOnce(); + mono = 73'000; + EXPECT_EQ(gc.runRegularRound().fence_outs, 1u); + EXPECT_TRUE(holder_fenced()); +} + } TEST(CASGCAckFloor, ExpiredMountFencedOutAndExcluded) @@ -1120,8 +1166,9 @@ TEST(CASGCAckFloor, ExpiredMountFencedOutAndExcluded) runExpiredMountFenceOutScenario(PoolConfig{.pool_prefix = "p", .server_root_id = "test"}); } -/// GC's fence-out threshold (`ttl + 5% drift allowance + one round's worth of renewal slack`, computed in -/// `Gc::runRegularRound`) never reads `PoolConfig::unsafe_remount_no_delay` -- that knob is consulted only +/// GC's fence-out threshold (`L + L/20 + step`; the holder's 100 ms lease is shorter than the observer's, so L +/// is the observer's own and the step is the renewal period) never reads `PoolConfig::unsafe_remount_no_delay` +/// -- that knob is consulted only /// by `Pool::mountWritable`'s own reclaim decision, never by GC's heartbeat-floor observation. Runs the /// IDENTICAL three-round scenario as `ExpiredMountFencedOutAndExcluded` with the knob turned on, and /// asserts the SAME round-by-round fence-out counts -- including the one-millisecond-short discriminator @@ -1142,6 +1189,16 @@ TEST(CASGcFenceOut, RebuildCountsASightingFromAfterItsRead) runSlowWalkSightingScenario(SightingCall::Rebuild); } +TEST(CASGcFenceOut, RoundWaitsForTheHoldersLongerLease) +{ + runLongerDeclaredLeaseScenario(SightingCall::Round); +} + +TEST(CASGcFenceOut, RebuildWaitsForTheHoldersLongerLease) +{ + runLongerDeclaredLeaseScenario(SightingCall::Rebuild); +} + /// fix-round F6 (author-review: `Gc`'s own `mono_ms_fn` used to default to the RAW static `Pool:: /// bootMs()`, bypassing the Pool's own injectable `config.boot_ms_fn` -- a time-controlled test can /// desync the mount side's fake clock from the GC side's real one). This mirrors @@ -1466,17 +1523,13 @@ TEST(CASGCCondemnMarker, SwallowedMarkerWriteCarriesEntryInsteadOfDeleting) - carries_before, 4u) << "every refused graduation must count one unconfirmed carry"; - /// Heal the backend: the carry-time retry publishes the marker, the entry confirms + graduates, and - /// the pipeline reclaims the blob. The retry stamped a round above the entry's, so the entry's marker - /// delete leaves it: an orphan over the absent body. - const uint64_t entry_round = e->condemn_round; + /// Heal the backend: the carry-time retry publishes the marker at a round above the entry's, the entry + /// confirms and graduates, and the pipeline reclaims the blob together with that re-stamped marker. backend->fail_meta_writes.store(false); EXPECT_TRUE(runRoundsUntilAbsent(store, gc, *backend, store->layout(), blob)); EXPECT_FALSE(currentEntryFor(*backend, store->layout(), blob).has_value()); - const auto orphan = loadMetaForTest(*backend, store->layout(), blob); - ASSERT_TRUE(orphan.has_value()); - EXPECT_EQ(orphan->meta.state, MetaState::Condemned); - EXPECT_GT(orphan->meta.condemn_round, entry_round); + EXPECT_FALSE(loadMetaForTest(*backend, store->layout(), blob).has_value()) + << "the re-stamped marker outlived its body"; } /// The healthy-path counterpart: with the condemn-time marker write landing normally, the gate must not @@ -1650,3 +1703,41 @@ TEST(CASGCRetire, OutcomeLogUnobservedConflictDoesNotReportItVanished) EXPECT_TRUE(refusal_reached) << "no round ever wrote an outcome log, so the arm was never reached"; } #endif + +/// The round passes the pool's own lease: the store's slot, renewed with that lease, is not named; +/// a holder renewing with 100 ms is named once across two rounds. +TEST(CASGCLeaseMismatch, RoundWarnsOnceAboutAHolderWithADifferentLease) +{ + auto backend = std::make_shared(); + auto store = Pool::open(backend, PoolConfig{.pool_prefix = "p", .server_root_id = "test"}); + const Layout & layout = store->layout(); + + const String holder_srid = "short-lease-server"; + CasRequests renewer_requests = openRequestsForTest(backend); + MountLeaseRenewer holder(renewer_requests, renewer_requests, layout, holder_srid, DB::UInt128(0x2222), + /*writer_epoch=*/1, + std::chrono::milliseconds(100), [] { return 1000u; }, [] { return 0u; }, {}, + std::chrono::milliseconds(0), [] { return 0u; }); + holder.start(); + + const RootNamespace ns{"00/aa@cas@"}; + const ManifestRef r = ref("srv-a:1", 1, 0xAA); + const UInt128 blob = DB::UInt128(7); + writeBlobBody(*backend, layout, blob); + writeManifestRaw(*backend, layout, ns, r, {blobEntryFor("a", blob)}); + publishCommittedTransition(*backend, layout, ns, "tbl", std::nullopt, r); + + uint64_t gc_now = 1'000'000; + uint64_t gc_mono = 0; + Gc gc(store, kGc, [&] { return gc_now; }, [&] { return gc_mono; }); + + ScopedLogCapture capture("CasHeartbeatFloor", "warning"); + gc.runRegularRound(); + gc_mono = 1; + gc.runRegularRound(); + + EXPECT_EQ(capture.count("declares a lease"), 1u) << capture.captured(); + EXPECT_EQ(capture.count("This server is configured with cas_mount_lease_ttl_ms=30000 but the holder of slot " + "short-lease-server (hostname="), 1u) << capture.captured(); + EXPECT_EQ(capture.count("declares a lease of 100 ms."), 1u) << capture.captured(); +} diff --git a/src/Disks/tests/gtest_cas_gc_meta_writer.cpp b/src/Disks/tests/gtest_cas_gc_meta_writer.cpp index 2c1f23ceaacd4..546bbfdf67116 100644 --- a/src/Disks/tests/gtest_cas_gc_meta_writer.cpp +++ b/src/Disks/tests/gtest_cas_gc_meta_writer.cpp @@ -78,7 +78,7 @@ TEST(CASGcMetaWriter, RealConfirmedMetaDeleteCompletesAcrossOwnerDestruction) auto gc = std::make_unique(store, DB::Cas::tests::u128Of(kGcId)); backend->arm(); - gc->metaWriterForTest().scheduleConfirmedMetaDelete(ref, /*condemn_round=*/1); + gc->metaWriterForTest().scheduleConfirmedMetaDelete(ref, /*newest_round=*/1); awaitLatchEntered(*backend); @@ -179,9 +179,9 @@ namespace { /// The delete job, scheduled the way `Gc::applyRedeleteOutcome` schedules it, and drained. -void runConfirmedMetaDelete(Gc & gc, const BlobRef & ref, uint64_t condemn_round) +void runConfirmedMetaDelete(Gc & gc, const BlobRef & ref, uint64_t newest_round) { - gc.metaWriterForTest().scheduleConfirmedMetaDelete(ref, condemn_round); + gc.metaWriterForTest().scheduleConfirmedMetaDelete(ref, newest_round); gc.metaWriterForTest().drain(); } @@ -739,9 +739,7 @@ TEST(CASGcMetaWriter, GraduationRefusesAnOlderRoundMarker) EXPECT_EQ(runGateRound(store, gc).graduated, 1u); EXPECT_EQ(runGateRound(store, gc).redeleted, 1u); EXPECT_FALSE(blobPresent(*backend, layout, x)); - const auto orphan = markerOf(*backend, layout, x); - ASSERT_TRUE(orphan.has_value()) << "the job for round " << c << " removed a newer marker"; - EXPECT_EQ(orphan->meta.condemn_round, c + 1); + EXPECT_FALSE(markerOf(*backend, layout, x).has_value()) << "the re-stamped marker outlived its body"; } TEST(CASGcMetaWriter, SparedThenRecondemnedBlobGraduates) @@ -775,3 +773,28 @@ TEST(CASGcMetaWriter, SparedThenRecondemnedBlobGraduates) return entry && entry->delete_pending; }, /*max_rounds*/ 2)) << "a spared, re-condemned blob never graduates"; } + +/// The condemn-marker write is lost, so the gate refuses and re-stamps the marker at the refusing round. +/// The redelete must remove that marker with the body; nothing else ever would. +TEST(CASGcMarker, RedeleteRemovesMarkerRestampedByGate) +{ + auto backend = std::make_shared(); + auto store = openGatePool(backend); + const Layout & layout = store->layout(); + const UInt128 hash = DB::UInt128(0x3604); + const BlobRef x = gateRef(hash); + Gc gc(store, kGateLeaderA); + uint64_t c = 0; + ASSERT_NO_FATAL_FAILURE(condemnBlobs(store, gc, *backend, "tbl", 1, {hash}, c)); + ASSERT_NO_FATAL_FAILURE(removeMarker(*backend, layout, x)); + + ASSERT_EQ(runGateRound(store, gc).graduated, 0u); + const auto restamped = markerOf(*backend, layout, x); + ASSERT_TRUE(restamped.has_value()); + ASSERT_EQ(restamped->meta.condemn_round, c + 1); + ASSERT_EQ(runGateRound(store, gc).graduated, 1u); + + ASSERT_EQ(runGateRound(store, gc).redeleted, 1u); + EXPECT_FALSE(blobPresent(*backend, layout, x)); + EXPECT_FALSE(markerOf(*backend, layout, x).has_value()) << "a Condemned marker outlived its body"; +} diff --git a/src/Disks/tests/gtest_cas_heartbeat.cpp b/src/Disks/tests/gtest_cas_heartbeat.cpp index c39935d7ca7f7..68f65658a6d57 100644 --- a/src/Disks/tests/gtest_cas_heartbeat.cpp +++ b/src/Disks/tests/gtest_cas_heartbeat.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -96,6 +97,9 @@ class RenewalScriptBackend : public InMemoryBackend ReturnThenCancel, ThrowBeforeThenLandAfterResolve, ThrowConnectHint, + ThrowAccessDenied, + ThrowMalformed, + RefusePrecondition, }; struct Attempt @@ -103,12 +107,22 @@ class RenewalScriptBackend : public InMemoryBackend String key; String bytes; std::optional expected; + uint64_t at_ms = 0; }; std::deque actions; std::vector attempts; std::function cancel_after_write; uint64_t read_calls = 0; + /// The injected boot clock, when a test asserts on when each attempt was sent. + const uint64_t * clock = nullptr; + /// While positive, a read of a mount slot fails with an access denial. + uint32_t refuse_reads = 0; + /// How many guarded mount writes had been sent when each read was made. + std::vector puts_at_read; + std::vector read_at_ms; + /// When set, a read of a mount slot that succeeds returns these bytes under the slot's real token. + std::optional mount_read_bytes; /// Only a GUARDED write of a mount slot is scripted; the fixture's own seeding and every other /// key reach the store untouched. @@ -119,7 +133,7 @@ class RenewalScriptBackend : public InMemoryBackend if (!expected_value || !key.ends_with("/mount")) return InMemoryBackend::write(key, bytes, expected_value, access); - attempts.push_back({key, bytes, expected_value}); + attempts.push_back({key, bytes, expected_value, clock ? *clock : 0}); const Action action = actions.empty() ? Action::Delegate : actions.front(); if (!actions.empty()) actions.pop_front(); @@ -133,6 +147,18 @@ class RenewalScriptBackend : public InMemoryBackend throw Poco::TimeoutException("connect timed out"); #endif } + if (action == Action::ThrowAccessDenied || action == Action::ThrowMalformed) + { +#if USE_AWS_S3 + if (action == Action::ThrowAccessDenied) + throw DB::S3Exception("injected renewal access denial", Aws::S3::S3Errors::ACCESS_DENIED); + throw DB::S3Exception("injected malformed renewal request", Aws::S3::S3Errors::INVALID_REQUEST); +#else + throw DB::Exception(DB::ErrorCodes::ABORTED, "store refusals need the S3 client"); +#endif + } + if (action == Action::RefusePrecondition) + return std::unexpected(RawConflict{}); if (action == Action::ThrowBefore || action == Action::ThrowBeforeThenLandAfterResolve) { @@ -156,7 +182,20 @@ class RenewalScriptBackend : public InMemoryBackend std::optional read(const String & key, TransportAccess & access) override { ++read_calls; + puts_at_read.push_back(attempts.size()); + read_at_ms.push_back(clock ? *clock : 0); + if (refuse_reads > 0 && key.ends_with("/mount")) + { + --refuse_reads; +#if USE_AWS_S3 + throw DB::S3Exception("injected read access denial", Aws::S3::S3Errors::ACCESS_DENIED); +#else + throw DB::Exception(DB::ErrorCodes::ABORTED, "store refusals need the S3 client"); +#endif + } std::optional result = InMemoryBackend::read(key, access); + if (result && mount_read_bytes && key.ends_with("/mount")) + result->bytes = *mount_read_bytes; if (pending && pending->key == key) { const Attempt delayed = *pending; @@ -181,6 +220,7 @@ MountRenewOperationEnvironment renewalEnvironment( .boot_ms = [&boot_ms] { return boot_ms; }, .live = live, .cancelled = cancelled, + .requests = {}, }; } @@ -948,6 +988,291 @@ TEST(CASHeartbeat, RenewalRetriesOneImmutableBodyAndAdoptsLostResponse) decodeMountLease(backend->attempts.front().bytes).write_attempt_id); } +/// The background renewal of an adopted slot, with the script and the request records empty. +class CASHeartbeatUntilDefinitive : public ::testing::Test +{ +protected: + void SetUp() override + { + backend->clock = &boot_ms; + seedOwnClaim(ops.op, layout, "test", uuid, 9, wall_ms, 1000); + renewer.start(); + clearRecords(); + } + + void clearRecords() + { + backend->attempts.clear(); + backend->read_calls = 0; + backend->puts_at_read.clear(); + backend->read_at_ms.clear(); + } + + MountRenewResult renew(const std::function & live = {}, const std::function & cancelled = {}) + { + MountRenewOperationEnvironment environment = renewalEnvironment(boot_ms, live, cancelled); + environment.refusal_pause_ms = kRefusalPauseMs; + return renewer.renewUntilDefinitive(environment); + } + + /// Rewrites the slot behind the renewer's back. + void rewriteSlot(const std::function & change) + { + const auto got = ops.op.read(layout.mountKey("test"), Retry::standard()); + MountLease lease = decodeMountLease(got->bytes); + change(lease); + ++lease.seq; + mustCommit(ops.op.replace(layout.mountKey("test"), encodeMountLease(lease), got->etag, Retry::standard()), "rewrite"); + clearRecords(); + } + + /// A refused precondition whose first two reads are refused: the third read decides, and it must + /// end the series as a loss without another PUT. + void expectLossObservedByTheReads() + { + backend->actions = {RenewalScriptBackend::Action::RefusePrecondition}; + backend->refuse_reads = 2; + + const MountRenewResult renewed = renew(); + + const DB::Exception failure = terminalException(renewed); + EXPECT_NE(failure.code(), DB::ErrorCodes::LOGICAL_ERROR); + EXPECT_EQ(renewer.state(), MountLeaseRenewerState::RenewalTerminal); + EXPECT_EQ(backend->attempts.size(), 1u); + EXPECT_EQ(backend->puts_at_read, (std::vector{1, 1, 1})); + } + + static constexpr uint64_t kRefusalPauseMs = 10'000; + static constexpr uint64_t kShortestSpacingMs = Retry::kDefinitiveAttemptSpacingMs - Retry::kDefinitiveAttemptSpacingMs / 5; + static constexpr uint64_t kLongestSpacingMs = Retry::kDefinitiveAttemptSpacingMs + Retry::kDefinitiveAttemptSpacingMs / 5; + + std::shared_ptr backend = std::make_shared(); + Layout layout{"pool"}; + const UInt128 uuid{0x1234}; + uint64_t wall_ms = 1000; + uint64_t boot_ms = 100; + Ops ops{backend, &boot_ms}; + MountLeaseRenewer renewer{ + ops.mount, ops.farewell, layout, "test", uuid, 9, std::chrono::milliseconds(1000), + [this] { return wall_ms; }, [] { return uint64_t{7}; }, {}, std::chrono::milliseconds(20), + [this] { return boot_ms; }}; +}; + +/// The background renewal outlives its lease with one tuple, and confirms its first start. +TEST_F(CASHeartbeatUntilDefinitive, RetriesOneTuplePastTheLease) +{ + for (int i = 0; i < 10; ++i) + backend->actions.push_back(RenewalScriptBackend::Action::ThrowBefore); + const uint64_t start = boot_ms; + + const MountRenewResult renewed = renew(); + + ASSERT_EQ(renewed.outcome, MountRenewOutcome::Committed); + EXPECT_EQ(renewed.attempt_start_boot_ms, start); + EXPECT_EQ(renewed.attempts_sent, 11u); + ASSERT_EQ(backend->attempts.size(), 11u); + EXPECT_GE(backend->attempts.back().at_ms, start + 8'000) << "the renewal kept retrying past the lease"; + for (size_t i = 1; i < backend->attempts.size(); ++i) + { + EXPECT_EQ(backend->attempts[i].bytes, backend->attempts[0].bytes); + EXPECT_EQ(backend->attempts[i].expected, backend->attempts[0].expected); + EXPECT_GE(backend->attempts[i].at_ms - backend->attempts[i - 1].at_ms, kShortestSpacingMs); + EXPECT_LE(backend->attempts[i].at_ms - backend->attempts[i - 1].at_ms, kLongestSpacingMs); + } +} + +/// A renewal confirmed after an outage longer than the lease stores an `expires_at_ms` already past. A GC +/// round that watched the previous token for its whole threshold does not fence the slot: the token changed. +TEST_F(CASHeartbeatUntilDefinitive, ALateCommitRestartsTheGcObservation) +{ + const uint64_t lease_ms = 1000; + const uint64_t cadence_ms = 1000; + const uint64_t threshold_ms = mountObservationThresholdMs(lease_ms, /*declared_lease_ms=*/lease_ms, cadence_ms); + MountObservationMap observations; + std::set lease_mismatch_warned; + const auto gc_round = [&] + { + return computeHeartbeatFloor( + ops.op, layout, wall_ms, [this] { return boot_ms; }, lease_ms, cadence_ms, observations, lease_mismatch_warned); + }; + ASSERT_EQ(gc_round().live, 1u); + clearRecords(); + for (int i = 0; i < 5; ++i) + backend->actions.push_back(RenewalScriptBackend::Action::ThrowBefore); + const uint64_t start = boot_ms; + + ASSERT_EQ(renew().outcome, MountRenewOutcome::Committed); + wall_ms += boot_ms - start; + + ASSERT_GE(boot_ms - start, threshold_ms) << "the GC watched the previous token for its whole threshold"; + const MountLease stored = decodeMountLease(ops.op.read(layout.mountKey("test"), Retry::standard())->bytes); + EXPECT_LT(stored.expires_at_ms, wall_ms) << "the commit stored a lease that already expired"; + const HeartbeatFloor floor = gc_round(); + EXPECT_EQ(floor.fenced_now, 0u); + EXPECT_EQ(floor.live, 1u); + EXPECT_FALSE(decodeMountLease(ops.op.read(layout.mountKey("test"), Retry::standard())->bytes).gc_fenced); +} + +#if USE_AWS_S3 +/// A refusal is not an answer about the slot: the next engine call sends the same tuple, one refusal +/// pause later, whatever the store refused for. +TEST_F(CASHeartbeatUntilDefinitive, ARefusedRenewalIsSentAgainAfterTheRefusalPause) +{ + backend->actions = {RenewalScriptBackend::Action::ThrowMalformed, RenewalScriptBackend::Action::ThrowAccessDenied, + RenewalScriptBackend::Action::Delegate}; + const uint64_t start = boot_ms; + + const MountRenewResult renewed = renew(); + + ASSERT_EQ(renewed.outcome, MountRenewOutcome::Committed); + EXPECT_EQ(renewed.attempt_start_boot_ms, start); + EXPECT_EQ(renewed.attempts_sent, 3u); + ASSERT_EQ(backend->attempts.size(), 3u); + for (size_t i = 1; i < backend->attempts.size(); ++i) + { + EXPECT_EQ(backend->attempts[i].bytes, backend->attempts[0].bytes); + EXPECT_EQ(backend->attempts[i].expected, backend->attempts[0].expected); + EXPECT_EQ(backend->attempts[i].at_ms - backend->attempts[i - 1].at_ms, kRefusalPauseMs) << "attempt " << i; + } + EXPECT_EQ(backend->read_calls, 0u) << "a refusal owes no read"; +} + +/// A refused precondition whose read failed is not a lost slot: only the read repeats, one spacing after +/// the write and one refusal pause after each refused read, and a read that finds the slot as the last +/// confirmed renewal left it sends the same tuple again. +TEST_F(CASHeartbeatUntilDefinitive, AfterACleanPreconditionRefusalOnlyTheReadRepeats) +{ + backend->actions = {RenewalScriptBackend::Action::RefusePrecondition, RenewalScriptBackend::Action::Delegate}; + backend->refuse_reads = 3; + + const MountRenewResult renewed = renew(); + + ASSERT_EQ(renewed.outcome, MountRenewOutcome::Committed); + ASSERT_EQ(backend->attempts.size(), 2u); + EXPECT_EQ(backend->attempts[1].bytes, backend->attempts[0].bytes); + EXPECT_EQ(backend->attempts[1].expected, backend->attempts[0].expected); + EXPECT_EQ(backend->puts_at_read, (std::vector{1, 1, 1, 1})) << "no PUT between the reads"; + ASSERT_EQ(backend->read_at_ms.size(), 4u); + EXPECT_GE(backend->read_at_ms[1] - backend->read_at_ms[0], kShortestSpacingMs); + EXPECT_LE(backend->read_at_ms[1] - backend->read_at_ms[0], kLongestSpacingMs); + EXPECT_EQ(backend->read_at_ms[2] - backend->read_at_ms[1], kRefusalPauseMs); + EXPECT_EQ(backend->read_at_ms[3] - backend->read_at_ms[2], kRefusalPauseMs); +} + +/// A read that finally observes a slot that moved ends the series as a loss, with no further PUT. +TEST_F(CASHeartbeatUntilDefinitive, AReadThatFindsAForeignHolderEndsTheSeries) +{ + rewriteSlot([](MountLease & lease) { lease.server_uuid = UInt128{0x9999}; }); + expectLossObservedByTheReads(); +} + +TEST_F(CASHeartbeatUntilDefinitive, AReadThatFindsOurSlotFencedEndsTheSeries) +{ + rewriteSlot([](MountLease & lease) { lease.gc_fenced = true; }); + expectLossObservedByTheReads(); +} + +TEST_F(CASHeartbeatUntilDefinitive, AReadThatFindsTheSlotDeletedEndsTheSeries) +{ + const auto got = ops.op.read(layout.mountKey("test"), Retry::standard()); + ASSERT_EQ(ops.op.remove(layout.mountKey("test"), got->etag, Retry::standard()), Removal::Removed); + clearRecords(); + expectLossObservedByTheReads(); +} + +/// Our token over bytes that are not our last confirmed body is not "the slot did not move". +TEST_F(CASHeartbeatUntilDefinitive, OurTokenOverOtherBytesEndsTheSeries) +{ + MountLease other = decodeMountLease(ops.op.read(layout.mountKey("test"), Retry::standard())->bytes); + other.write_attempt_id = UInt128{0xABCD}; + ++other.seq; + backend->mount_read_bytes = encodeMountLease(other); + clearRecords(); + expectLossObservedByTheReads(); +} + +/// A stop between engine calls ends the series as cancelled: nothing of ours is in flight, the renewer +/// stays active and its farewell still hands the slot back. +TEST_F(CASHeartbeatUntilDefinitive, AStopBetweenCallsEndsTheSeriesAsCancelled) +{ + backend->actions = {RenewalScriptBackend::Action::ThrowMalformed}; + const auto stopped = [&] { return !backend->attempts.empty(); }; + + /// No refusal pause: the farewell has to fit in the lease the last commit confirmed. + const MountRenewResult renewed + = renewer.renewUntilDefinitive(renewalEnvironment(boot_ms, [&] { return !stopped(); }, stopped)); + + EXPECT_EQ(renewed.outcome, MountRenewOutcome::NotAttempted); + EXPECT_EQ(renewer.state(), MountLeaseRenewerState::Active); + EXPECT_EQ(backend->attempts.size(), 1u); + renewer.release(); + const MountLease left = decodeMountLease(ops.op.read(layout.mountKey("test"), Retry::standard())->bytes); + EXPECT_EQ(left.min_active_build_sequence, std::numeric_limits::max()) << "the farewell landed"; +} + +/// A stop while only the read of a refused precondition repeats ends the series as cancelled with no +/// further PUT, and without a farewell: the slot may have moved to a holder entitled to it. +TEST_F(CASHeartbeatUntilDefinitive, AStopDuringTheReadsEndsTheSeriesWithoutAFarewell) +{ + backend->actions = {RenewalScriptBackend::Action::RefusePrecondition}; + backend->refuse_reads = 5; + const auto stopped = [&] { return backend->read_calls >= 2; }; + + const MountRenewResult renewed = renew([&] { return !stopped(); }, stopped); + + const DB::Exception failure = terminalException(renewed); + EXPECT_NE(failure.message().find("(cancelled,"), String::npos) << failure.message(); + EXPECT_EQ(renewer.state(), MountLeaseRenewerState::RenewalTerminal); + EXPECT_FALSE(renewer.canRelease()); + EXPECT_EQ(backend->attempts.size(), 1u); + EXPECT_EQ(backend->puts_at_read, (std::vector{1, 1})); +} + +/// A lost liveness without a stop ends the reads as a loss. +TEST_F(CASHeartbeatUntilDefinitive, ALostLivenessDuringTheReadsEndsTheSeriesAsALoss) +{ + backend->actions = {RenewalScriptBackend::Action::RefusePrecondition}; + backend->refuse_reads = 5; + + const MountRenewResult renewed = renew([&] { return backend->read_calls < 2; }); + + const DB::Exception failure = terminalException(renewed); + EXPECT_NE(failure.message().find("(fence_or_lifecycle_lost,"), String::npos) << failure.message(); + EXPECT_EQ(renewer.state(), MountLeaseRenewerState::RenewalTerminal); + EXPECT_EQ(backend->attempts.size(), 1u); + EXPECT_EQ(backend->puts_at_read, (std::vector{1, 1})); +} + +/// A renewal that ends after an earlier engine call sent an attempt reports that attempt. +TEST_F(CASHeartbeatUntilDefinitive, AGiveUpAfterARefusedCallCountsItsAttempts) +{ + backend->actions = {RenewalScriptBackend::Action::ThrowMalformed}; + + const MountRenewResult renewed = renew([&] { return backend->attempts.empty(); }); + + ASSERT_EQ(renewed.outcome, MountRenewOutcome::Terminal); + EXPECT_EQ(renewed.attempts_sent, 1u); + EXPECT_TRUE(renewed.sent_any); + const DB::Exception e = terminalException(renewed); + EXPECT_NE(e.message().find("at least one attempt sent"), String::npos) << e.message(); +} + +/// A refusal that follows an unclear attempt does not end the engine call: the read settles the +/// unclear attempt, and our attempt that landed meanwhile is adopted, not taken for another writer. +TEST_F(CASHeartbeatUntilDefinitive, ARefusalAfterAnUnclearAttemptIsSettledInsideTheCall) +{ + backend->actions = {RenewalScriptBackend::Action::ThrowBeforeThenLandAfterResolve, + RenewalScriptBackend::Action::ThrowAccessDenied}; + + const MountRenewResult renewed = renew(); + + ASSERT_EQ(renewed.outcome, MountRenewOutcome::Committed); + EXPECT_TRUE(renewed.resolved_by_read); + EXPECT_EQ(renewed.attempts_sent, 2u); + EXPECT_EQ(backend->attempts.size(), 2u); +} +#endif + #if USE_AWS_S3 TEST(CASHeartbeat, RenewalOverConnectFailuresRecoversWithoutASettleRead) { @@ -1035,6 +1360,40 @@ TEST(CASHeartbeat, CancellationBeforeSendIsNotAttemptedAndAllowsRelease) EXPECT_EQ(renewer.state(), MountLeaseRenewerState::Released); } +/// The stop lands after the renewal began and before its admission. Nothing was sent, so the renewer stays +/// releasable and the teardown still writes the farewell. +TEST(CASHeartbeat, CancellationRacingAdmissionIsNotAttemptedAndAllowsRelease) +{ + auto backend = std::make_shared(); + Layout layout("pool"); + uint64_t wall_ms = 1000; + uint64_t boot_ms = 100; + bool cancelled = false; + Ops ops(backend, &boot_ms); + seedOwnClaim(ops.op, layout, "test", UInt128{1}, 9, wall_ms, 1000); + MountLeaseRenewer renewer( + ops.mount, ops.farewell, layout, "test", UInt128{1}, 9, std::chrono::milliseconds(1000), + [&] { return wall_ms; }, [] { return uint64_t{0}; }, {}, std::chrono::milliseconds(20), + [&] { return boot_ms; }); + renewer.start(); + backend->attempts.clear(); + backend->read_calls = 0; + const auto stop_then_refuse = [&] + { + cancelled = true; + return false; + }; + const MountRenewResult result = renewer.renew(renewalEnvironment( + boot_ms, /*live=*/stop_then_refuse, /*cancelled=*/[&] { return cancelled; })); + EXPECT_EQ(result.outcome, MountRenewOutcome::NotAttempted); + EXPECT_EQ(result.failure, nullptr); + ASSERT_EQ(renewer.state(), MountLeaseRenewerState::Active); + EXPECT_TRUE(backend->attempts.empty()); + EXPECT_NO_THROW(renewer.release()); + EXPECT_EQ(decodeMountLease(ops.op.read(layout.mountKey("test"), Retry::standard())->bytes).min_active_build_sequence, + std::numeric_limits::max()); +} + TEST(CASHeartbeat, CancellationAfterSendIsTerminalAndForbidsRelease) { auto backend = std::make_shared(); @@ -1057,6 +1416,7 @@ TEST(CASHeartbeat, CancellationAfterSendIsTerminalAndForbidsRelease) renewalEnvironment(boot_ms, /*live=*/[&] { return !cancelled; }, /*cancelled=*/[&] { return cancelled; })); const DB::Exception failure = terminalException(result); EXPECT_EQ(failure.code(), DB::ErrorCodes::NETWORK_ERROR); + EXPECT_NE(failure.message().find("(cancelled,"), String::npos) << failure.message(); EXPECT_TRUE(result.sent_any); EXPECT_EQ(backend->read_calls, 0u) << "post-write cancellation must not start a diagnostic read"; EXPECT_EQ(renewer.state(), MountLeaseRenewerState::RenewalTerminal); diff --git a/src/Disks/tests/gtest_cas_layout.cpp b/src/Disks/tests/gtest_cas_layout.cpp index 1006e87188a2a..48f42a6ff6cf4 100644 --- a/src/Disks/tests/gtest_cas_layout.cpp +++ b/src/Disks/tests/gtest_cas_layout.cpp @@ -3,6 +3,9 @@ #include #include #include "cas_test_helpers.h" +#include +#include +#include using namespace DB::Cas; @@ -380,3 +383,28 @@ TEST(CASLayout, BlobKeyRoundTripsThroughParse) EXPECT_EQ(*parsed_meta, ref); EXPECT_FALSE(layout.parseBlobKey("pool0/blobs/unknown-algo/aa/aa00").has_value()); /// foreign => nullopt } + +TEST(CASLayout, StagingKeyCarriesTheWriterEpoch) +{ + const Layout layout("p"); + EXPECT_EQ(layout.stagingPrefix("a/b"), "p/staging/a/b/"); + const String key = layout.stagingKey("a/b", 7, "x.tmp"); + EXPECT_EQ(key, "p/staging/a/b/7/x.tmp"); + EXPECT_EQ(Layout::stagedEpochOf(std::string_view(key).substr(layout.stagingPrefix("a/b").size())), 7u); + + const std::vector>> cases{ + {"x.tmp", 0}, /// flat key written before keys carried an epoch + {"0/x.tmp", 0}, + {"007/x.tmp", 7}, + {"18446744073709551615/x.tmp", std::numeric_limits::max()}, + {"18446744073709551616/x.tmp", std::nullopt}, /// overflow + {"", std::nullopt}, + {"7/", std::nullopt}, /// a folder marker + {"/x.tmp", std::nullopt}, + {"a/x.tmp", std::nullopt}, /// a nested member's key + {"+7/x.tmp", std::nullopt}, + {"7/a/x.tmp", std::nullopt}, /// deeper than one epoch directory + }; + for (const auto & [rest, expected] : cases) + EXPECT_EQ(Layout::stagedEpochOf(rest), expected) << rest; +} diff --git a/src/Disks/tests/gtest_cas_lifecycle_snapshot.cpp b/src/Disks/tests/gtest_cas_lifecycle_snapshot.cpp index ea504c8946414..c1817a76108cb 100644 --- a/src/Disks/tests/gtest_cas_lifecycle_snapshot.cpp +++ b/src/Disks/tests/gtest_cas_lifecycle_snapshot.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -235,3 +236,54 @@ TEST(CASLifecycleSnapshot, NaturalIdentityLostMatchesThrowDetail) << "snapshot detail and the typed error must not drift\n detail: " << snap.detail << "\n thrown: " << thrown; } + +/// An expired lease while the lifecycle stays `Live`: `not_live` / `lease_expired`, `since` is before the +/// deadline, and a deadline back in the future returns the row to `live`. +TEST(CASLifecycleSnapshot, ExpiredLeaseIsNotLiveUntilRestored) +{ + auto storage = openSnapshotStorage(); + commitOnePart(*storage); + auto pool = storage->store(); + const auto wall_now_s = [] + { + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + }; + + pool->setMountDeadline(pool->bootMsNow() - 5'000); + const CasLifecycleSnapshot expired = storage->lifecycleSnapshot(); + const int64_t after_s = wall_now_s(); + + EXPECT_EQ(expired.lifecycle, "not_live"); + EXPECT_EQ(expired.reason, "lease_expired"); + EXPECT_GT(static_cast(expired.since), 0); + EXPECT_LE(static_cast(expired.since), after_s - 4); + EXPECT_EQ(pool->lifecycle(), PoolLifecycle::Live) << "the lifecycle enum is not touched"; + + pool->setMountDeadline(pool->bootMsNow() + 60'000); + const CasLifecycleSnapshot restored = storage->lifecycleSnapshot(); + EXPECT_EQ(restored.lifecycle, "live"); + EXPECT_TRUE(restored.reason.empty()) << restored.reason; + EXPECT_EQ(restored.since, 0); +} + +/// The row is `not_live` from the moment the lease has too little time left to admit a write, before the +/// deadline itself, and `since` moves with the deadline. +TEST(CASLifecycleSnapshot, NotLiveOnceTheLeaseCannotAdmitAWrite) +{ + auto storage = openSnapshotStorage(); + commitOnePart(*storage); + auto pool = storage->store(); + + pool->setMountDeadline(pool->bootMsNow() + 3'000); + const CasLifecycleSnapshot closing = storage->lifecycleSnapshot(); + EXPECT_TRUE(pool->mayMutate()) << "the deadline has not passed"; + EXPECT_EQ(closing.lifecycle, "not_live"); + EXPECT_EQ(closing.reason, "lease_expired"); + + pool->setMountDeadline(pool->bootMsNow() - 5'000); + const CasLifecycleSnapshot expired = storage->lifecycleSnapshot(); + const int64_t moved_s = static_cast(closing.since) - static_cast(expired.since); + EXPECT_GE(moved_s, 7); + EXPECT_LE(moved_s, 9); +} diff --git a/src/Disks/tests/gtest_cas_mount.cpp b/src/Disks/tests/gtest_cas_mount.cpp index 161ffbcc5296b..779bb3dc344fb 100644 --- a/src/Disks/tests/gtest_cas_mount.cpp +++ b/src/Disks/tests/gtest_cas_mount.cpp @@ -9,17 +9,22 @@ #include #include +#include #include #include +#include #include #include +#include #include +#include #include #include #include namespace DB::ErrorCodes { + extern const int ABORTED; extern const int BAD_ARGUMENTS; extern const int CORRUPTED_DATA; extern const int FILE_DOESNT_EXIST; @@ -28,6 +33,7 @@ namespace DB::ErrorCodes namespace ProfileEvents { + extern const Event CASIdentityLost; extern const Event CASMountLeaseLost; extern const Event CASMountExclusivityViolation; } @@ -119,10 +125,10 @@ class OwnerConflictRevealsManifestBackend : public InMemoryBackend const String & key, const String & bytes, const std::optional & expected_value, TransportAccess & access) override { - if (!fired && !expected_value && key == "p/gc/server-roots/root/x/owner") + if (!fired && !expected_value && key == "p/gc/server-roots/rootx/owner") { fired = true; - InMemoryBackend::write("p/cas/manifests/root/x/table/debris", "x", std::nullopt, access); + InMemoryBackend::write("p/cas/manifests/rootx/table/debris", "x", std::nullopt, access); return std::unexpected(RawConflict{}); } return InMemoryBackend::write(key, bytes, expected_value, access); @@ -170,6 +176,51 @@ class OwnerRaceBackend : public InMemoryBackend UInt128 winner; }; +/// Owner ETags derived from the bytes, as S3 derives them: equal bytes bring back an equal ETag. +class ContentEtagOwnerBackend : public InMemoryBackend +{ +public: + std::optional read(const String & key, TransportAccess & access) override + { + auto raw = InMemoryBackend::read(key, access); + if (raw && isOwner(key)) + raw->value = contentValue(raw->bytes); + return raw; + } + + std::optional head(const String & key, TransportAccess & access) override + { + auto meta = InMemoryBackend::head(key, access); + if (meta && isOwner(key)) + if (const auto raw = InMemoryBackend::read(key, access)) + meta->value = contentValue(raw->bytes); + return meta; + } + + std::expected write(const String & key, const String & bytes, + const std::optional & expected_value, TransportAccess & access) override + { + if (!isOwner(key)) + return InMemoryBackend::write(key, bytes, expected_value, access); + std::optional minted_expected; + if (expected_value) + { + const auto current = InMemoryBackend::read(key, access); + if (!current || contentValue(current->bytes) != *expected_value) + return std::unexpected(RawConflict{}); + minted_expected = current->value; + } + const auto written = InMemoryBackend::write(key, bytes, minted_expected, access); + if (!written) + return written; + return contentValue(bytes); + } + +private: + static bool isOwner(const String & key) { return key.ends_with("/owner"); } + static String contentValue(const String & bytes) { return "content-" + DB::Cas::tests::hexOf(bytes); } +}; + /// Refuses the FIRST write of the epoch key after installing a competing allocator's own epoch, so the /// absent-epoch decision has to be made a second time. `reveal_owned_work` decides whether owned work /// becomes visible at that same instant -- the fact the second decision must re-establish. @@ -202,6 +253,131 @@ class EpochConflictBackend : public InMemoryBackend bool reveal_owned_work; }; +/// Between this start's epoch allocation and its claim read, a second start of the same server +/// allocates the next epoch and installs its live body. Armed by the test after the bootstrap open. +class NewerStartBackend : public InMemoryBackend +{ +public: + String epoch_key; + String mount_key; + bool arm_on_next_epoch_write = false; + std::optional newer_epoch; + + std::expected write( + const String & key, const String & bytes, + const std::optional & expected_value, TransportAccess & access) override + { + auto result = InMemoryBackend::write(key, bytes, expected_value, access); + if (arm_on_next_epoch_write && key == epoch_key && result) + { + arm_on_next_epoch_write = false; + install_on_next_mount_read = true; + } + return result; + } + + std::optional read(const String & key, TransportAccess & access) override + { + if (install_on_next_mount_read && key == mount_key) + { + install_on_next_mount_read = false; + installNewerStart(access); + } + return InMemoryBackend::read(key, access); + } + +private: + bool install_on_next_mount_read = false; + + void installNewerStart(TransportAccess & access) + { + const auto epoch = InMemoryBackend::read(epoch_key, access); + const auto mount = InMemoryBackend::read(mount_key, access); + if (!epoch || !mount) + throw std::runtime_error("NewerStartBackend: a control object is missing"); + ServerEpoch counter = decodeServerEpoch(epoch->bytes); + newer_epoch = counter.next_writer_epoch; + ++counter.next_writer_epoch; + if (!InMemoryBackend::write(epoch_key, encodeServerEpoch(counter), epoch->value, access)) + throw std::runtime_error("NewerStartBackend: the epoch bump conflicted"); + const MountLease newer{ + .server_uuid = UInt128(1), + .writer_epoch = *newer_epoch, + .hostname = "newer-start", + .pid = 2, + .started_at_ms = 1, + .seq = 1, + .expires_at_ms = 4'000'000'000'000, + .write_attempt_id = UInt128{0xB}}; + if (!InMemoryBackend::write(mount_key, encodeMountLease(newer), mount->value, access)) + throw std::runtime_error("NewerStartBackend: the mount install conflicted"); + } +}; + +/// Records every write of `key`; while `lose_first` is set, the first one times out without applying. +class UnappliedFirstWriteBackend : public InMemoryBackend +{ +public: + String key; + bool lose_first = false; + std::vector>> writes; + + std::expected write( + const String & k, const String & bytes, + const std::optional & expected_value, TransportAccess & access) override + { + if (k == key) + { + writes.emplace_back(bytes, expected_value); + if (std::exchange(lose_first, false)) + throw Poco::TimeoutException("UnappliedFirstWriteBackend: response lost, the attempt did not apply"); + } + return InMemoryBackend::write(k, bytes, expected_value, access); + } +}; + +/// Applies the first conditional write of `key` after arming, then loses its response. +class LandedReplaceAckLostOnceBackend : public InMemoryBackend +{ +public: + String key; + bool armed = false; + + std::expected write( + const String & k, const String & bytes, + const std::optional & expected_value, TransportAccess & access) override + { + if (armed && expected_value && k == key) + { + armed = false; + (void)InMemoryBackend::write(k, bytes, expected_value, access); + throw Poco::TimeoutException("LandedReplaceAckLostOnceBackend: simulated lost PUT response"); + } + return InMemoryBackend::write(k, bytes, expected_value, access); + } +}; + +/// On the first write of `key` after `competitor` is set, runs `competitor` (another allocator of the +/// same uuid) to completion, then times out our write without applying it. +class TwinAllocatorBackend : public InMemoryBackend +{ +public: + String key; + std::function competitor; + + std::expected write( + const String & k, const String & bytes, + const std::optional & expected_value, TransportAccess & access) override + { + if (competitor && k == key) + { + std::exchange(competitor, {})(); + throw Poco::TimeoutException("TwinAllocatorBackend: response lost, our attempt did not apply"); + } + return InMemoryBackend::write(k, bytes, expected_value, access); + } +}; + /// Counts every request that reaches the store, per primitive, so a test can pin how many a protocol /// step costs rather than only what it produced. class RequestCountingBackend final : public InMemoryBackend @@ -251,36 +427,6 @@ class RenewalLogBackend final : public InMemoryBackend } }; -class ScopedRenewalLogCapture -{ -public: - explicit ScopedRenewalLogCapture(const String & level) - : logger(getLogger("CasMountLeaseRenewer")) - , channel(new Poco::StreamChannel(stream)) - , old_channel(logger->getChannel(), /*shared=*/true) - , old_level(logger->getLevel()) - { - logger->setChannel(channel.get()); - logger->setLevel(level); - } - - ~ScopedRenewalLogCapture() - { - logger->setChannel(old_channel); - logger->setLevel(old_level); - } - - String captured() const { return stream.str(); } - -private: - LoggerPtr logger; - std::ostringstream stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - Poco::AutoPtr channel; - /// A real reference (shared=true), so the parked previous channel cannot die while ours is installed. - Poco::AutoPtr old_channel; - int old_level; -}; - size_t countRenewalLogText(const String & haystack, std::string_view needle) { size_t count = 0; @@ -335,7 +481,7 @@ TEST(CASMountAudit, RenewalDefaultLogsAreBounded) auto backend = std::make_shared(); auto boot_ms = std::make_shared>(100); auto store = open_store(backend, boot_ms, "renewal-log-silent"); - ScopedRenewalLogCapture capture("information"); + DB::Cas::tests::ScopedLoggerCapture capture("CasMountLeaseRenewer", "information"); EXPECT_NO_THROW(store->renewWatermarkOnce()); EXPECT_EQ(countRenewalLogText(capture.captured(), "CAS mount renewal"), 0u); } @@ -344,7 +490,7 @@ TEST(CASMountAudit, RenewalDefaultLogsAreBounded) auto backend = std::make_shared(); auto boot_ms = std::make_shared>(100); auto store = open_store(backend, boot_ms, "renewal-log-recovered"); - ScopedRenewalLogCapture capture("information"); + DB::Cas::tests::ScopedLoggerCapture capture("CasMountLeaseRenewer", "information"); backend->throw_before_next_overwrite = true; EXPECT_NO_THROW(store->renewWatermarkOnce()); const String output = capture.captured(); @@ -357,7 +503,7 @@ TEST(CASMountAudit, RenewalDefaultLogsAreBounded) auto backend = std::make_shared(); auto boot_ms = std::make_shared>(100); auto store = open_store(backend, boot_ms, "renewal-log-debug"); - ScopedRenewalLogCapture capture("debug"); + DB::Cas::tests::ScopedLoggerCapture capture("CasMountLeaseRenewer", "debug"); backend->throw_before_next_overwrite = true; EXPECT_NO_THROW(store->renewWatermarkOnce()); EXPECT_EQ(countRenewalLogText(capture.captured(), "physical retry attempt 2"), 1u); @@ -367,7 +513,7 @@ TEST(CASMountAudit, RenewalDefaultLogsAreBounded) auto backend = std::make_shared(); auto boot_ms = std::make_shared>(100); auto store = open_store(backend, boot_ms, "renewal-log-fenced"); - ScopedRenewalLogCapture capture("information"); + DB::Cas::tests::ScopedLoggerCapture capture("CasMountLeaseRenewer", "information"); /// The lease was claimed at boot 100 with the 1000 ms TTL above, so it expires at 1100. The /// fence admits only while the remaining time strictly clears the safety margin plus whatever /// the attempt reserves, so exactly `margin` remaining (with the reservation on top) refuses. @@ -382,7 +528,7 @@ TEST(CASMountAudit, RenewalDefaultLogsAreBounded) TEST(CASServerRootId, ValidationAcceptsCleanPathsRejectsBad) { EXPECT_NO_THROW(validateServerRootId("replica-a")); - EXPECT_NO_THROW(validateServerRootId("shard-01/replica-a")); + EXPECT_THROW(validateServerRootId("shard-01/replica-a"), DB::Exception); EXPECT_THROW(validateServerRootId(""), DB::Exception); EXPECT_THROW(validateServerRootId("/replica"), DB::Exception); EXPECT_THROW(validateServerRootId("replica/"), DB::Exception); @@ -391,6 +537,30 @@ TEST(CASServerRootId, ValidationAcceptsCleanPathsRejectsBad) EXPECT_THROW(validateServerRootId("a/_files/b"), DB::Exception); } +/// Decommission and the staging sweep select a member's objects by key prefix, so a member `a/b` would be +/// drained by a decommission of `a`. An id with `/` is refused even when its owner object already exists. +TEST(CASServerRootId, ExistingIdWithSlashRefusesToOpen) +{ + auto b = std::make_shared(); + const Layout l("p"); + DB::Cas::tests::seedPoolMetaForRestart(*b); + { + Ops seed(b); + mustCommit(seed.op.create(l.ownerKey("shard-01/replica-a"), + encodeOwner(OwnerObject{.server_uuid = UInt128(1), .retired_at_ms = std::nullopt}), + Retry::standard()), "owner of an existing nested member"); + } + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::BAD_ARGUMENTS, "new without '/'", [&] + { + Pool::open(b, PoolConfig{.pool_prefix = "p", .server_id = UInt128(1), .server_root_id = "shard-01/replica-a"}); + }); + Ops check(b); + EXPECT_FALSE(check.op.head(l.mountKey("shard-01/replica-a"), Retry::standard()).has_value()) + << "a refused open must not claim the mount slot"; + EXPECT_FALSE(check.op.head(l.epochKey("shard-01/replica-a"), Retry::standard()).has_value()) + << "a refused open must not mint an epoch"; +} + TEST(CASServerRoot, KeysAndCodecsRoundTrip) { Layout layout("p"); @@ -482,7 +652,141 @@ TEST(CASServerRootClaim, TombstonedSameOwnerFailsClosed) { EXPECT_EQ(e.code(), DB::ErrorCodes::CORRUPTED_DATA); EXPECT_NE(e.message().find("decommissioned"), String::npos) << e.message(); - EXPECT_EQ(e.message().find("owned by a different server"), String::npos) << e.message(); + } +} + +TEST(CASServerRootClaim, FirstClaimWritesLifeOne) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + EXPECT_EQ(claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()), 1u); + const OwnerObject stored = decodeOwner(ops.op.read(l.ownerKey("r"), Retry::standard())->bytes); + EXPECT_EQ(stored.owner_life, 1u); + EXPECT_FALSE(stored.retirement_verified); + /// A restart under the same owner writes nothing and reads the same life. + const Etag before = currentEtag(ops.op, l.ownerKey("r")); + EXPECT_EQ(claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()), 1u); + EXPECT_EQ(currentEtag(ops.op, l.ownerKey("r")), before); +} + +TEST(CASServerRootClaim, ClaimOverA2664OwnerReadsLifeZero) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + mustCommit(ops.op.create(l.ownerKey("r"), encodeOwner(OwnerObject{.server_uuid = UInt128(1), .retired_at_ms = std::nullopt}), Retry::standard()), + "owner without a life"); + EXPECT_EQ(claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()), 0u); +} + +TEST(CASServerRootClaim, DecommissionPolicyAcceptsARetiredOwnerOfTheSameUuid) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + mustCommit(ops.op.create(l.ownerKey("r"), encodeOwner(OwnerObject{ + .server_uuid = UInt128(1), .retired_at_ms = 1752537600000ULL, .owner_life = 2}), Retry::standard()), "tombstone"); + DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::CORRUPTED_DATA, [&] + { + claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation(), RetiredOwnerPolicy::Refuse); + }); + EXPECT_EQ(claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation(), + RetiredOwnerPolicy::AcceptForDecommission), 2u); + /// A retired owner of ANOTHER uuid is still foreign under either policy; the refusal also names a duplicate id. + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "same ", [&] + { + claimOwnerOrThrow(ops.op, l, "r", UInt128(2), emptyCatalogObservation(), RetiredOwnerPolicy::AcceptForDecommission); + }); +} + +/// The owner appears between the claim's read and its create: the conflict re-read applies the same retired +/// policy as the first read. +TEST(CASServerRootClaim, ConflictRereadAppliesTheRetiredPolicy) +{ + for (const RetiredOwnerPolicy policy : + {RetiredOwnerPolicy::Refuse, RetiredOwnerPolicy::AcceptForDecommission, RetiredOwnerPolicy::RejoinIfVerified}) + { + SCOPED_TRACE(static_cast(policy)); + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + bool raced = false; + b->onBeforeWrite(l.ownerKey("r"), [&] + { + if (std::exchange(raced, true)) + return; + Ops racer(b); + mustCommit(racer.op.create(l.ownerKey("r"), encodeOwner(OwnerObject{ + .server_uuid = UInt128(1), .retired_at_ms = 1752537600000ULL, .owner_life = 2}), Retry::standard()), + "racing tombstone"); + }); + if (policy != RetiredOwnerPolicy::AcceptForDecommission) + { + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "decommission is not finished", [&] + { + claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation(), policy); + }); + } + else + { + EXPECT_EQ(claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation(), policy), 2u); + } + EXPECT_TRUE(raced); + } +} + +/// A start over an unverified tombstone is refused and writes neither the counter nor the slot. +TEST(CASMount, PoolOpenRefusesAnUnverifiedRetirement) +{ + auto b = std::make_shared(); + Layout l("p"); + { auto first = Pool::open(b, PoolConfig{.pool_prefix = "p", .server_id = UInt128(1), .server_root_id = "r"}); } + Ops ops(b); + const auto owner = ops.op.read(l.ownerKey("r"), Retry::standard()); + OwnerObject tombstone = decodeOwner(owner->bytes); + tombstone.retired_at_ms = 1; + tombstone.owner_life += 1; + tombstone.retirement_verified = false; + mustCommit(ops.op.replace(l.ownerKey("r"), encodeOwner(tombstone), owner->etag, Retry::standard()), "tombstone"); + const Etag epoch_before = currentEtag(ops.op, l.epochKey("r")); + const Etag mount_before = currentEtag(ops.op, l.mountKey("r")); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "decommission is not finished", [&] + { + Pool::open(b, PoolConfig{.pool_prefix = "p", .server_id = UInt128(1), .server_root_id = "r"}); + }); + EXPECT_EQ(currentEtag(ops.op, l.epochKey("r")), epoch_before); + EXPECT_EQ(currentEtag(ops.op, l.mountKey("r")), mount_before); +} + +/// Each identity refusal names the supported recovery: the decommission verb for a dead foreign owner, +/// a new id for a retired one. Neither advises a hand edit of the owner object as the first option. +TEST(CASServerRootClaim, RefusalsNameTheSupportedRecovery) +{ + auto b = std::make_shared(); + const Layout l("p"); + Ops ops(b); + claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "SYSTEM CAS DROP POOL MEMBER 'r'", [&] + { + claimOwnerOrThrow(ops.op, l, "r", UInt128(2), emptyCatalogObservation()); + }); + + mustCommit(ops.op.create(l.ownerKey("t"), encodeOwner(OwnerObject{ + .server_uuid = UInt128(1), + .retired_at_ms = 1752537600000ULL, + }), Retry::standard()), "tombstoned owner"); + try + { + claimOwnerOrThrow(ops.op, l, "t", UInt128(1), emptyCatalogObservation()); + FAIL() << "expected a tombstoned owner claim to fail closed"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::CORRUPTED_DATA); + EXPECT_NE(e.message().find("was decommissioned"), String::npos) << e.message(); + EXPECT_NE(e.message().find("decommission is not finished"), String::npos) << e.message(); + EXPECT_EQ(e.message().find("clear the owner object"), String::npos) << e.message(); } } @@ -492,8 +796,8 @@ TEST(CASServerRootEpoch, AllocatorIsMonotoneAndSurvivesMountConcept) Layout l("r"); Ops ops(b); claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()); - const uint64_t e1 = allocateWriterEpoch(ops.op, l, "r", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()); - const uint64_t e2 = allocateWriterEpoch(ops.op, l, "r", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()); + const uint64_t e1 = allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()); + const uint64_t e2 = allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()); EXPECT_GE(e1, 1u); // 0 is a reserved sentinel EXPECT_GT(e2, e1); // strictly increasing @@ -501,7 +805,120 @@ TEST(CASServerRootEpoch, AllocatorIsMonotoneAndSurvivesMountConcept) /// so the removal is a no-op that touches nothing. ASSERT_FALSE(ops.op.head(l.mountKey("r"), Retry::standard()).has_value()); EXPECT_EQ(ops.op.removeCurrent(l.mountKey("r"), Retry::standard()), Removal::Gone); - EXPECT_GT(allocateWriterEpoch(ops.op, l, "r", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), e2); + EXPECT_GT(allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()), e2); +} + +/// Our counter write is lost while a twin of the same uuid allocates the same next epoch. Its counter +/// body must not read back as our landed write, on both the create and the replace path. +TEST(CASServerRootEpoch, TwinAllocatorsNeverShareAnEpochAfterALostWrite) +{ + for (const bool counter_exists : {false, true}) + { + SCOPED_TRACE(counter_exists ? "replace" : "create"); + auto b = std::make_shared(); + Layout l("p"); + Ops ours(b); + Ops twin(b); + b->key = l.epochKey("r"); + if (counter_exists) + ASSERT_EQ(allocateWriterEpoch(ours.op, l, "r", emptyCatalogObservation()), 1u); + + uint64_t twin_epoch = 0; + b->competitor = [&] + { + twin_epoch = allocateWriterEpoch(twin.op, l, "r", emptyCatalogObservation()); + }; + const uint64_t our_epoch + = allocateWriterEpoch(ours.op, l, "r", emptyCatalogObservation()); + ASSERT_FALSE(b->competitor) << "the twin never ran"; + ASSERT_NE(twin_epoch, 0u); + EXPECT_NE(our_epoch, twin_epoch); + EXPECT_EQ(decodeServerEpoch(ours.op.read(l.epochKey("r"), Retry::standard())->bytes).next_writer_epoch, + std::max(our_epoch, twin_epoch) + 1); + } +} + +/// A resend of one allocation decision carries that decision's nonce, so our own landed write with a +/// lost response is recognized as ours instead of costing a second epoch. +TEST(CASServerRootEpoch, LandedCounterWriteWithLostResponseIsOurs) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + b->key = l.epochKey("r"); + ASSERT_EQ(allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()), 1u); + + b->armed = true; + EXPECT_EQ(allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()), 2u); + ASSERT_FALSE(b->armed) << "the lost response was never injected"; + EXPECT_EQ(decodeServerEpoch(ops.op.read(l.epochKey("r"), Retry::standard())->bytes).next_writer_epoch, 3u); +} + +/// An unapplied write with an unclear outcome is reissued inside the same decision: every physical +/// attempt must carry the same body, nonce included, under the same precondition. +TEST(CASServerRootEpoch, ReissuedCounterWriteKeepsItsNonce) +{ + for (const bool counter_exists : {false, true}) + { + SCOPED_TRACE(counter_exists ? "replace" : "create"); + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + b->key = l.epochKey("r"); + if (counter_exists) + ASSERT_EQ(allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()), 1u); + + b->writes.clear(); + b->lose_first = true; + const uint64_t expected_epoch = counter_exists ? 2 : 1; + EXPECT_EQ(allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()), + expected_epoch); + ASSERT_FALSE(b->lose_first) << "the lost write was never injected"; + ASSERT_GE(b->writes.size(), 2u) << "the unclear write was never reissued"; + for (const auto & [bytes, precondition] : b->writes) + { + EXPECT_EQ(bytes, b->writes.front().first); + EXPECT_EQ(precondition, b->writes.front().second); + } + const ServerEpoch stored = decodeServerEpoch(ops.op.read(l.epochKey("r"), Retry::standard())->bytes); + EXPECT_EQ(stored.next_writer_epoch, expected_epoch + 1); + EXPECT_TRUE(stored.allocation_nonce.has_value()); + } +} + +TEST(CASServerRootEpoch, EveryAllocationWritesAFreshNonce) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + std::vector nonces; + for (int i = 0; i < 2; ++i) + { + (void)allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()); + const std::optional nonce + = decodeServerEpoch(ops.op.read(l.epochKey("r"), Retry::standard())->bytes).allocation_nonce; + ASSERT_TRUE(nonce.has_value()); + nonces.push_back(*nonce); + } + EXPECT_NE(nonces[0], nonces[1]); +} + +/// No writer emits a zero counter; an allocation over one would hand out writer_epoch 0. +TEST(CASServerRootEpoch, ZeroCounterFailsAllocationWithoutWriting) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + mustCommit(ops.op.create(l.epochKey("r"), encodeServerEpoch(ServerEpoch{.next_writer_epoch = 0}), Retry::standard()), + "zero counter"); + const auto before = ops.op.read(l.epochKey("r"), Retry::standard()); + ASSERT_TRUE(before.has_value()); + + DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::CORRUPTED_DATA, [&] + { + (void)allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()); + }); + EXPECT_EQ(ops.op.read(l.epochKey("r"), Retry::standard())->bytes, before->bytes); } /// Phase C (spec rev.4): an ABSENT epoch object over a PRESENT mount object means durable epoch @@ -516,7 +933,7 @@ TEST(CASMount, EpochRemintOverExistingMountRefuses) ASSERT_EQ(claimMount(ops.op, l, "r", UInt128(1), /*our_epoch=*/1, /*now_ms=*/1000, /*ttl_ms=*/30000).kind, MountClaimResult::Claimed); /// The epoch object is ABSENT (never created in this sequence) while the mount exists: - EXPECT_THROW(allocateWriterEpoch(ops.op, l, "r", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), DB::Exception); /// CORRUPTED_DATA + EXPECT_THROW(allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()), DB::Exception); /// CORRUPTED_DATA } TEST(CASMount, EpochRemintAuthoritativeAbsenceMints) @@ -525,8 +942,8 @@ TEST(CASMount, EpochRemintAuthoritativeAbsenceMints) Layout l("p"); Ops ops(b); claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()); - EXPECT_EQ(allocateWriterEpoch(ops.op, l, "r", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), 1u); /// fresh root: both control objects absent - EXPECT_EQ(allocateWriterEpoch(ops.op, l, "r", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), 2u); /// epoch present now: normal conditional bump, no probe + EXPECT_EQ(allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()), 1u); /// fresh root: both control objects absent + EXPECT_EQ(allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()), 2u); /// epoch present now: normal conditional bump, no probe } /// The probe outcome gates the mint: anything short of authoritative KeyAbsent fails closed. @@ -544,35 +961,7 @@ TEST(CASMount, EpochRemintIndeterminateProbeFailsClosed) Layout l("p"); Ops ops(b); claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()); - EXPECT_THROW(allocateWriterEpoch(ops.op, l, "r", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), DB::Exception); -} - -/// Decommission over a TERMINAL (expired/fenced) mount with a lost epoch object proceeds and mints -/// an epoch DISTINCT from the surviving mount's — the same-pair state is unrepresentable. -TEST(CASMount, DecommissionRemintOverTerminalMountMintsDistinctEpoch) -{ - auto b = std::make_shared(); - Layout l("p"); - Ops ops(b); - claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()); - ASSERT_EQ(claimMount(ops.op, l, "r", UInt128(1), /*our_epoch=*/3, /*now_ms=*/1000, /*ttl_ms=*/100).kind, - MountClaimResult::Claimed); - /// now_ms=5000: the ttl_ms=100 lease above is long expired -> terminal. - EXPECT_EQ(allocateWriterEpoch(ops.op, l, "r", EpochMintPolicy::DecommissionRecovery, /*now_ms=*/5000, emptyCatalogObservation()), 4u); -} - -/// Decommission over a LIVE mount with a lost epoch refuses — the blind bypass would recreate the -/// forbidden pair (codex round-3 finding 1) and defeat CASDecommission.RefusesLiveMember. -TEST(CASMount, DecommissionRemintOverLiveMountRefuses) -{ - auto b = std::make_shared(); - Layout l("p"); - Ops ops(b); - claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()); - ASSERT_EQ(claimMount(ops.op, l, "r", UInt128(1), /*our_epoch=*/1, /*now_ms=*/1000, /*ttl_ms=*/30000).kind, - MountClaimResult::Claimed); - EXPECT_THROW(allocateWriterEpoch(ops.op, l, "r", EpochMintPolicy::DecommissionRecovery, /*now_ms=*/2000, emptyCatalogObservation()), - DB::Exception); /// ABORTED: live member + EXPECT_THROW(allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()), DB::Exception); } /// The steady-state path (epoch object PRESENT) must never pay the probe — pins the zero @@ -593,9 +982,9 @@ TEST(CASMount, EpochBumpWithPresentEpochIssuesNoProbe) Layout l("p"); Ops ops(b); claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()); - EXPECT_EQ(allocateWriterEpoch(ops.op, l, "r", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), 1u); /// bootstrap: ONE probe (absent-epoch branch) + EXPECT_EQ(allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()), 1u); /// bootstrap: ONE probe (absent-epoch branch) const int probes_after_bootstrap = b->probes; - EXPECT_EQ(allocateWriterEpoch(ops.op, l, "r", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), 2u); /// epoch present: normal conditional bump... + EXPECT_EQ(allocateWriterEpoch(ops.op, l, "r", emptyCatalogObservation()), 2u); /// epoch present: normal conditional bump... EXPECT_EQ(b->probes, probes_after_bootstrap) << "...must not probe the mount key"; } @@ -606,7 +995,17 @@ TEST(CASServerRootClaim, MissingOwnerOverNonEmptyRootIsCorrupted) Ops ops(b); /// Simulate existing data without an owner (identity lost): plant a key under roots//. mustCommit(ops.op.create(l.serverRootDataPrefix("r") + "some-data", "x", Retry::standard()), "root debris"); - EXPECT_THROW(claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()), DB::Exception); + try + { + claimOwnerOrThrow(ops.op, l, "r", UInt128(1), emptyCatalogObservation()); + FAIL() << "the claim over an ownerless non-empty subtree must be refused"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::CORRUPTED_DATA) << e.message(); + EXPECT_NE(e.message().find("refusing to re-claim"), String::npos) << e.message(); + EXPECT_NE(e.message().find("Configure a new for this disk"), String::npos) << e.message(); + } } TEST(CASServerRootSafety, EveryCatalogLifecycleStateBlocksOwnerAndEpochRecreation) @@ -618,12 +1017,16 @@ TEST(CASServerRootSafety, EveryCatalogLifecycleStateBlocksOwnerAndEpochRecreatio const ObserveRefCatalog observe = [catalog] { return catalog; }; Ops owner_ops(std::make_shared()); - EXPECT_THROW(claimOwnerOrThrow(owner_ops.op, layout, "root/x", UInt128{1}, observe), DB::Exception); + DB::Cas::tests::expectThrowsCodeWithMessage( + DB::ErrorCodes::CORRUPTED_DATA, "identity lost over existing data", [&] + { + claimOwnerOrThrow(owner_ops.op, layout, "root/x", UInt128{1}, observe); + }); EXPECT_FALSE(owner_ops.op.head(layout.ownerKey("root/x"), Retry::standard()).has_value()); Ops epoch_ops(std::make_shared()); EXPECT_THROW(allocateWriterEpoch( - epoch_ops.op, layout, "root/x", EpochMintPolicy::NormalMount, 0, observe), DB::Exception); + epoch_ops.op, layout, "root/x", observe), DB::Exception); EXPECT_FALSE(epoch_ops.op.head(layout.epochKey("root/x"), Retry::standard()).has_value()); } } @@ -647,9 +1050,9 @@ TEST(CASServerRootSafety, OpaqueStreamAndStateDebrisAloneDoesNotBlockRecreation) mustCommit(ops.op.create(layout.refCkptKey(dead), "debris", Retry::standard()), "ckpt debris"); mustCommit(ops.op.create(layout.namespaceFileKey(dead, "f"), "debris", Retry::standard()), "ns-file debris"); - EXPECT_NO_THROW(claimOwnerOrThrow(ops.op, layout, "root/x", UInt128{1}, emptyCatalogObservation())); + EXPECT_NO_THROW(claimOwnerOrThrow(ops.op, layout, "rootx", UInt128{1}, emptyCatalogObservation())); EXPECT_EQ(allocateWriterEpoch( - ops.op, layout, "root/x", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), 1u); + ops.op, layout, "rootx", emptyCatalogObservation()), 1u); } TEST(CASServerRootSafety, ManifestAndLooseRootDebrisStillBlockRecreation) @@ -661,10 +1064,13 @@ TEST(CASServerRootSafety, ManifestAndLooseRootDebrisStillBlockRecreation) { Ops ops(std::make_shared()); mustCommit(ops.op.create(key, "x", Retry::standard()), "blocking debris"); - EXPECT_THROW(claimOwnerOrThrow( - ops.op, layout, "root/x", UInt128{1}, emptyCatalogObservation()), DB::Exception); + DB::Cas::tests::expectThrowsCodeWithMessage( + DB::ErrorCodes::CORRUPTED_DATA, "identity lost over existing data", [&] + { + claimOwnerOrThrow(ops.op, layout, "root/x", UInt128{1}, emptyCatalogObservation()); + }); EXPECT_THROW(allocateWriterEpoch( - ops.op, layout, "root/x", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), DB::Exception); + ops.op, layout, "root/x", emptyCatalogObservation()), DB::Exception); } } @@ -676,9 +1082,13 @@ TEST(CASServerRootSafety, UnreadableCatalogNeverFallsBackToPhysicalGuesses) { throw DB::Exception(DB::ErrorCodes::CORRUPTED_DATA, "injected unreadable catalog"); }; - EXPECT_THROW(claimOwnerOrThrow(ops.op, layout, "root/x", UInt128{1}, unreadable), DB::Exception); + DB::Cas::tests::expectThrowsCodeWithMessage( + DB::ErrorCodes::CORRUPTED_DATA, "injected unreadable catalog", [&] + { + claimOwnerOrThrow(ops.op, layout, "root/x", UInt128{1}, unreadable); + }); EXPECT_THROW(allocateWriterEpoch( - ops.op, layout, "root/x", EpochMintPolicy::NormalMount, 0, unreadable), DB::Exception); + ops.op, layout, "root/x", unreadable), DB::Exception); EXPECT_FALSE(ops.op.head(layout.ownerKey("root/x"), Retry::standard()).has_value()); EXPECT_FALSE(ops.op.head(layout.epochKey("root/x"), Retry::standard()).has_value()); } @@ -694,9 +1104,9 @@ TEST(CASServerRootSafety, OwnerConflictRecomputesTheWholeEmptinessBundle) DB::Cas::tests::expectThrowsCodeWithMessage( DB::ErrorCodes::CORRUPTED_DATA, "newly visible owned work blocks recreation", - [&] { claimOwnerOrThrow(ops.op, layout, "root/x", UInt128{1}, emptyCatalogObservation()); }); + [&] { claimOwnerOrThrow(ops.op, layout, "rootx", UInt128{1}, emptyCatalogObservation()); }); EXPECT_TRUE(backend->fired); - EXPECT_FALSE(ops.op.head(layout.ownerKey("root/x"), Retry::standard()).has_value()); + EXPECT_FALSE(ops.op.head(layout.ownerKey("rootx"), Retry::standard()).has_value()); } TEST(CASServerRootSafety, EpochConflictRecomputesTheWholeEmptinessBundle) @@ -705,7 +1115,7 @@ TEST(CASServerRootSafety, EpochConflictRecomputesTheWholeEmptinessBundle) Ops ops(backend); const Layout layout("p"); EXPECT_THROW(allocateWriterEpoch( - ops.op, layout, "root/x", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), DB::Exception); + ops.op, layout, "root/x", emptyCatalogObservation()), DB::Exception); EXPECT_TRUE(backend->fired); ASSERT_TRUE(backend->winner_installed); const auto epoch = ops.op.read(layout.epochKey("root/x"), Retry::standard()); @@ -976,7 +1386,13 @@ TEST(CASMountAwaitExpiry, LiveRenewingTwinTimesOutAsDoubleStart) { wall += ms; mono += ms; - ASSERT_EQ(claimMount(ops.op, l, "r", UInt128(1), 7, wall, 100).kind, MountClaimResult::Claimed); + const auto got = ops.op.read(l.mountKey("r"), Retry::standard()); + ASSERT_TRUE(got.has_value()); + MountLease renewed = decodeMountLease(got->bytes); + ++renewed.seq; + renewed.expires_at_ms = wall + 100; + mustCommit(ops.op.replace(l.mountKey("r"), encodeMountLease(renewed), got->etag, Retry::standard()), + "twin renewal"); }; const auto r = claimMountAwaitingExpiry( @@ -987,62 +1403,32 @@ TEST(CASMountAwaitExpiry, LiveRenewingTwinTimesOutAsDoubleStart) namespace { -/// fix-round F5 harness: makes the mount key vanish to EVERY read, unconditionally, while the real -/// underlying object stays put -- forcing `claimMount`'s own read to take the absent-slot race -/// branch every call (its create then conflicts against the real, still-present object, returning -/// `LiveDoubleStart` with no incarnation -- that branch deliberately leaves `.etag` unset, -/// since no re-read was done). That in turn forces `claimMountAwaitingExpiry`'s fallback re-read, -/// which ALSO sees the slot as vanished -- deterministically reproducing "the slot vanished between -/// claimMount's own read and ours" on EVERY loop iteration, not just a lucky one-shot race. -class AlwaysVanishesBackend final : public DB::Cas::Backend +/// Hides the watched key from every read while the object stays underneath, so `claimMount`'s create +/// conflicts with nothing observed and the observation loop's re-read finds the slot absent every time. +class AlwaysVanishesBackend final : public InMemoryBackend { public: - explicit AlwaysVanishesBackend(std::shared_ptr inner_) : inner(std::move(inner_)) {} String watched_key; - bool supportsListTokens() const override { return inner->supportsListTokens(); } - - /// The fault is on the read primitive, which is the only way anything now reaches the store. std::optional read(const String & key, TransportAccess & access) override { if (key == watched_key) return std::nullopt; - return inner->read(key, access); - } - std::optional head(const String & key, TransportAccess & access) override { return inner->head(key, access); } - RawListPage list(const String & prefix, const String & cursor, size_t limit, TransportAccess & access) override { return inner->list(prefix, cursor, limit, access); } - RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override { return inner->remove(key, expected_value, access); } - void removeManyWriteOnce(const std::vector & keys, TransportAccess & access) override { inner->removeManyWriteOnce(keys, access); } - size_t bulkDeleteKeyLimit() const override { return inner->bulkDeleteKeyLimit(); } - std::expected write(const String & key, const String & bytes, - const std::optional & expected_value, TransportAccess & access) override - { - return inner->write(key, bytes, expected_value, access); + return InMemoryBackend::read(key, access); } - std::unique_ptr stream(const String & key, TransportAccess & access) override { return inner->stream(key, access); } - void publish(const BlobPublishRequest & request, TransportAccess & access) override { inner->publish(request, access); } - Dialect dialect() const override { return inner->dialect(); } - -private: - std::shared_ptr inner; }; } -/// fix-round F5 (author-review: `!got -> continue` in the observation loop, with no sleep and outside -/// the restart limit, spins `get`/`claimMount`/`put` at backend RTT under persistent slot churn). A -/// backend that makes the mount slot look vanished to every GET must still terminate (bounded restarts, -/// not an infinite loop) AND must pace itself (the injected `sleep_fn` must actually fire) rather than -/// busy-spin. +/// A slot deleted under the observation loop (a 26.6.4 decommission still deletes it) is re-claimed on the +/// same paced, bounded restart budget as a renewing holder, not spun at backend RTT. TEST(CASMountAwaitExpiry, PersistentSlotVanishPacesAndBoundsRestartsInsteadOfSpinning) { - auto inner = std::make_shared(); + auto vanishing = std::make_shared(); Layout l("p"); - Ops inner_ops(inner); - /// A real slot exists underneath (uuid 1, epoch 7) so `claimMount`'s absent-slot create - /// genuinely conflicts every time (never accidentally re-mints). - ASSERT_EQ(claimMount(inner_ops.op, l, "r", UInt128(1), 7, /*now*/ 1000, /*ttl*/ 100).kind, MountClaimResult::Claimed); - - auto vanishing = std::make_shared(inner); + { + Ops seed(vanishing); + ASSERT_EQ(claimMount(seed.op, l, "r", UInt128(1), 7, /*now*/ 1000, /*ttl*/ 100).kind, MountClaimResult::Claimed); + } vanishing->watched_key = l.mountKey("r"); Ops ops(vanishing); @@ -1055,11 +1441,10 @@ TEST(CASMountAwaitExpiry, PersistentSlotVanishPacesAndBoundsRestartsInsteadOfSpi const auto r = claimMountAwaitingExpiry( ops.op, l, "r", UInt128(1), /*our_epoch*/ 8, now_fn, mono_fn, /*ttl*/ 100, /*poll*/ 20, sleep_fn); - EXPECT_EQ(r.kind, MountClaimResult::LiveDoubleStart) << "must terminate (bounded), not loop forever"; - EXPECT_GT(sleeps, 0) << "a persistently vanishing slot must still pace via sleep_fn, not busy-spin"; - /// The real epoch-7 lease is untouched -- every create attempt against it genuinely conflicts - /// (the object is still there), so it is never accidentally re-minted over. - EXPECT_EQ(decodeMountLease(inner_ops.op.read(l.mountKey("r"), Retry::standard())->bytes).writer_epoch, 7u); + EXPECT_EQ(r.kind, MountClaimResult::LiveDoubleStart); + EXPECT_GT(sleeps, 0); + vanishing->watched_key.clear(); + EXPECT_EQ(decodeMountLease(ops.op.read(l.mountKey("r"), Retry::standard())->bytes).writer_epoch, 7u); } TEST(CASMountAwaitExpiry, ForeignUuidFailsClosedImmediately) @@ -1082,29 +1467,71 @@ TEST(CASMountAwaitExpiry, ForeignUuidFailsClosedImmediately) EXPECT_EQ(sleeps, 0); // never waits across UUIDs } -/// rev.6: the predecessor's own stamped `expires_at_ms` (however skewed) is NEVER consulted for the -/// reclaim decision any more — the wait is bounded purely by OUR OWN `ttl_ms`-derived threshold. A -/// prior incarnation minted with an absurdly large `ttl` (so its own stamp claims aliveness for -/// ~100000ms) still reclaims within the SAME small threshold as any other case, because that stamp is -/// never read for timing. -TEST(CASMountAwaitExpiry, SkewedFarFutureExpiryHasNoEffectOnObservationThreshold) +namespace +{ +/// Restart reclaim as this server with a 30 s lease and a 1 s poll. Returns the observer clock at the +/// reclaim and records every threshold `on_wait_start` reports. +uint64_t reclaimAsShortLeaseObserver(Ops & ops, const Layout & l, std::vector & thresholds) { - auto b = std::make_shared(); - Layout l("p"); - Ops ops(b); - ASSERT_EQ(claimMount(ops.op, l, "r", UInt128(1), 7, /*now*/ 1000, /*ttl*/ 100000).kind, MountClaimResult::Claimed); - - uint64_t wall = 1000; uint64_t mono = 0; - auto now_fn = [&] { return wall; }; - auto mono_fn = [&] { return mono; }; - auto sleep_fn = [&](uint64_t ms) { wall += ms; mono += ms; }; - const auto r = claimMountAwaitingExpiry( - ops.op, l, "r", UInt128(1), /*our_epoch*/ 8, now_fn, mono_fn, /*ttl*/ 100, /*poll*/ 20, sleep_fn); + ops.op, l, "r", UInt128(1), /*our_epoch*/ 8, [] { return uint64_t{1000}; }, [&] { return mono; }, + /*ttl*/ 30'000, /*poll*/ 1'000, [&](uint64_t ms) { mono += ms; }, + [&](const MountLease &, uint64_t threshold_ms) { thresholds.push_back(threshold_ms); }); EXPECT_EQ(r.kind, MountClaimResult::Claimed); - EXPECT_LE(mono, 100u + 100u / 20 + 20u + 20u); // bounded by OUR threshold, not the predecessor's stamp - EXPECT_EQ(decodeMountLease(ops.op.read(l.mountKey("r"), Retry::standard())->bytes).writer_epoch, 8u); // reclaimed + EXPECT_EQ(r.prior, MountPriorState::UncleanObserved); + return mono; +} +} + +TEST(CASMountAwaitExpiry, HolderDeclaringALongerLeaseIsWaitedOutInFull) +{ + Ops ops(std::make_shared()); + Layout l("p"); + ASSERT_EQ(claimMount(ops.op, l, "r", UInt128(1), 7, /*now*/ 1000, /*ttl*/ 60'000).kind, MountClaimResult::Claimed); + std::vector thresholds; + EXPECT_EQ(reclaimAsShortLeaseObserver(ops, l, thresholds), 64'000u) + << "reclaimed before the predecessor's declared lease ran out"; + EXPECT_EQ(thresholds, std::vector{64'000}); +} + +/// A shorter declared lease (10 s), a missing start stamp, equal stamps, an expiry before the start. +TEST(CASMountAwaitExpiry, ShorterOrUnusableDeclaredLeaseWaitsTheObserversOwn) +{ + const std::vector> stamps{{1'000, 11'000}, {0, 60'000}, {5'000, 5'000}, {70'000, 10'000}}; + for (const auto & [started, expires] : stamps) + { + SCOPED_TRACE(fmt::format("started_at_ms={} expires_at_ms={}", started, expires)); + Ops ops(std::make_shared()); + Layout l("p"); + const MountLease m{.server_uuid = UInt128(1), .writer_epoch = 7, .hostname = "h", .pid = 1, + .started_at_ms = started, .seq = 1, .expires_at_ms = expires, .write_attempt_id = UInt128(1)}; + mustCommit(ops.op.create(l.mountKey("r"), encodeMountLease(m), Retry::standard()), "seeded mount"); + std::vector thresholds; + EXPECT_EQ(reclaimAsShortLeaseObserver(ops, l, thresholds), 33'000u); + EXPECT_EQ(thresholds, std::vector{32'500}); + } +} + +/// The certificates of death need no wait, whatever lease the body declares. +TEST(CASMountAwaitExpiry, FencedAndFarewellSlotsReclaimWithoutWaitingWhateverTheyDeclare) +{ + for (const bool fenced : {true, false}) + { + Ops ops(std::make_shared()); + Layout l("p"); + MountLease m{.server_uuid = UInt128(1), .writer_epoch = 7, .hostname = "h", .pid = 1, + .started_at_ms = 1000, .seq = 1, .expires_at_ms = 61'000, .write_attempt_id = UInt128(1)}; + m.gc_fenced = fenced; + m.min_active_build_sequence = fenced ? 0 : std::numeric_limits::max(); + mustCommit(ops.op.create(l.mountKey("r"), encodeMountLease(m), Retry::standard()), "seeded mount"); + int sleeps = 0; + const auto r = claimMountAwaitingExpiry(ops.op, l, "r", UInt128(1), 8, [] { return uint64_t{1000}; }, + [] { return uint64_t{0}; }, 30'000, 1'000, [&](uint64_t) { ++sleeps; }); + EXPECT_EQ(r.kind, MountClaimResult::Claimed); + EXPECT_EQ(r.prior, fenced ? MountPriorState::Fenced : MountPriorState::Clean); + EXPECT_EQ(sleeps, 0); + } } TEST(CASMountClaim, UnsafeAuthorizationIsTokenExact) @@ -1116,10 +1543,16 @@ TEST(CASMountClaim, UnsafeAuthorizationIsTokenExact) const Etag stale = ops.op.read(l.mountKey("r"), Retry::standard())->etag; /// `Etag` equality compares (key, value), and it is minted only through the request planes -- there - /// is no cross-key comparison to exercise here. Build the stale token by refreshing the SAME slot a - /// second time (same uuid, same epoch): `stale`, read before this refresh, is then a genuinely stale - /// token for the slot's CURRENT value, without touching an unrelated key. - ASSERT_EQ(claimMount(ops.op, l, "r", UInt128(1), 7, /*now*/ 1000, /*ttl*/ 30000).kind, MountClaimResult::Claimed); + /// is no cross-key comparison to exercise here. Build the stale token by renewing the SAME slot once: + /// `stale`, read before this renewal, is then a genuinely stale token for the slot's CURRENT value. + { + const auto got = ops.op.read(l.mountKey("r"), Retry::standard()); + ASSERT_TRUE(got.has_value()); + MountLease renewed = decodeMountLease(got->bytes); + ++renewed.seq; + mustCommit(ops.op.replace(l.mountKey("r"), encodeMountLease(renewed), got->etag, Retry::standard()), + "holder renewal"); + } const Etag current = ops.op.read(l.mountKey("r"), Retry::standard())->etag; /// A stale token is refused: nothing authorizes a reclaim over a slot that moved. @@ -1211,6 +1644,40 @@ TEST(CASMountStartup, WriterEpochStrictlyIncreasesAcrossReopen) EXPECT_GT(e2, e1); } +TEST(CASMountStartup, DelayedStartOverANewerIncarnationFailsWithoutWaiting) +{ + auto b = std::make_shared(); + const Layout l("p"); + b->epoch_key = l.epochKey("r"); + b->mount_key = l.mountKey("r"); + { + /// Bootstrap: owner, counter and a clean farewell of epoch 1. + auto first = Pool::open(b, PoolConfig{.pool_prefix = "p", .server_id = UInt128(1), .server_root_id = "r"}); + } + b->arm_on_next_epoch_write = true; + + /// Shared, heap-owned: the Pool may outlive this frame through a background publish. + auto fake_boot = std::make_shared>(0); + auto waits = std::make_shared>(0); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::ABORTED, "newer than the writer_epoch", [&] + { + (void)Pool::open(b, PoolConfig{ + .pool_prefix = "p", .server_id = UInt128(1), .server_root_id = "r", + .boot_ms_fn = [fake_boot] { return fake_boot->load(); }, + .wait_sleep_fn = [fake_boot, waits](uint64_t ms) + { + *fake_boot += ms; + ++(*waits); + }}); + }); + ASSERT_TRUE(b->newer_epoch.has_value()) << "the newer start was never installed"; + EXPECT_EQ(waits->load(), 0u) << "a newer incarnation is refused, not observed"; + Ops ops(b); + const MountLease slot = decodeMountLease(ops.op.read(l.mountKey("r"), Retry::standard())->bytes); + EXPECT_EQ(slot.writer_epoch, *b->newer_epoch); + EXPECT_EQ(slot.write_attempt_id, UInt128{0xB}); +} + TEST(CASMountStartup, FreshWritablePoolBootstrapsAnExplicitEmptyCatalog) { auto backend = std::make_shared(); @@ -1502,22 +1969,25 @@ namespace /// Rev.6 §token-stability observation removed the wall clock from the fence DECISION; `kNowMs` below /// is threaded through only as `computeHeartbeatFloor`'s audit-only `now_ms`. constexpr uint64_t kNowMs = 1'000'000; -/// The fence-out threshold measured on the LEADER's OWN monotonic clock (`mono_ms_fn`), independent -/// of any lease's stamped `expires_at_ms`. -constexpr uint64_t kStableThresholdMs = 10'000; +/// The observing leader's own lease and step, and the threshold they give a body that declares nothing. +constexpr uint64_t kObserverLeaseMs = 30'000; +constexpr uint64_t kCadenceMs = 1'000; +constexpr uint64_t kStableThresholdMs = 32'500; +constexpr std::string_view kLeaseMismatchMarker = "declares a lease"; /// Seed one mount body under mountKey(srid) via the on-storage codec — the same interface the renewer /// writes through. MountLease seedMount( CasOperation & op, const Layout & l, const String & srid, - uint64_t expires_at_ms, bool gc_fenced, uint64_t min_active_build_sequence, uint64_t seq = 1) + uint64_t expires_at_ms, bool gc_fenced, uint64_t min_active_build_sequence, uint64_t seq = 1, + uint64_t started_at_ms = kNowMs) { MountLease m; m.server_uuid = UInt128(srid.back()); // distinct per srid; content is irrelevant to the gate m.writer_epoch = 1; m.hostname = "h-" + srid; m.pid = 100; - m.started_at_ms = kNowMs; + m.started_at_ms = started_at_ms; m.seq = seq; m.expires_at_ms = expires_at_ms; m.min_active_build_sequence = min_active_build_sequence; @@ -1560,8 +2030,9 @@ TEST(CASHeartbeatFloor, FirstSightNeverFencesEvenIfStampLooksExpired) seedMount(ops.op, l, "s1", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); MountObservationMap obs; + std::set warned; const HeartbeatFloor floor = computeHeartbeatFloor(ops.op, l, /*now_ms*/ kNowMs, frozenClock(0), - kStableThresholdMs, obs); + kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(floor.fenced_now, 0u); EXPECT_EQ(floor.live, 1u); @@ -1577,7 +2048,8 @@ TEST(CASHeartbeatFloor, StableIncarnationPastThresholdIsFenced) seedMount(ops.op, l, "s1", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); MountObservationMap obs; - const HeartbeatFloor floor_before = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kStableThresholdMs, obs); + std::set warned; + const HeartbeatFloor floor_before = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(floor_before.fenced_now, 0u); const MountLease before = decodeMountLease(ops.op.read(l.mountKey("s1"), Retry::standard())->bytes); @@ -1585,7 +2057,7 @@ TEST(CASHeartbeatFloor, StableIncarnationPastThresholdIsFenced) /// No renewal in between: the SAME incarnation, observed since mono 0, is now stable for the full /// threshold on the leader's own clock. const HeartbeatFloor floor2 = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(kStableThresholdMs), - kStableThresholdMs, obs); + kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(floor2.fenced_now, 1u); EXPECT_EQ(floor2.fenced_srids, std::vector{"s1"}); @@ -1602,7 +2074,8 @@ TEST(CASHeartbeatFloor, RenewalBetweenRoundsRestartsObservation) seedMount(ops.op, l, "s1", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); MountObservationMap obs; - computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kStableThresholdMs, obs); + std::set warned; + computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); ASSERT_TRUE(obs.contains("s1")); const Etag first_etag = obs.at("s1").etag; @@ -1611,7 +2084,7 @@ TEST(CASHeartbeatFloor, RenewalBetweenRoundsRestartsObservation) EXPECT_NE(renewed_etag, first_etag); const HeartbeatFloor floor2 = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(kStableThresholdMs), - kStableThresholdMs, obs); + kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(floor2.fenced_now, 0u); ASSERT_TRUE(obs.contains("s1")); @@ -1619,36 +2092,51 @@ TEST(CASHeartbeatFloor, RenewalBetweenRoundsRestartsObservation) EXPECT_EQ(obs.at("s1").first_seen_mono_ms, kStableThresholdMs); } -/// fix-round F7 (author-review: `Gc::mount_obs` not pruned for srids gone from LIST -> slow unbounded -/// growth on a long-lived leader, worsened by pool-member decommission). A srid whose `/mount` key is -/// removed ENTIRELY (not merely fenced/terminated -- those already `obs.erase` themselves mid-loop) is -/// never visited by a later LIST pass again, so its observation entry must be pruned at end-of-round, -/// not linger in `obs` forever. -TEST(CASHeartbeatFloor, UnseenSridPrunedFromObservationMap) +/// A slot that turns terminal leaves the observation map in the walk that sees it. +TEST(CASHeartbeatFloor, TerminalSlotLeavesTheObservationMap) { auto b = std::make_shared(); Layout l("p"); Ops ops(b); seedMount(ops.op, l, "s1", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); seedMount(ops.op, l, "s2", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); - MountObservationMap obs; - computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kStableThresholdMs, obs); + std::set warned; + computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); ASSERT_TRUE(obs.contains("s1")); ASSERT_TRUE(obs.contains("s2")); - /// s2's `/mount` key is removed entirely -- e.g. `SYSTEM CAS DROP POOL MEMBER` -- so - /// no future LIST pass will ever visit it again. s1 renews (a live renewer would), so its OWN - /// observation restarts and it stays `live` -- isolating this test to the pruning behavior alone, - /// not confounding it with s1 also becoming fence-eligible (which would erase its `obs` entry too, - /// for an unrelated reason). renewMount(ops.op, l, "s1"); - ASSERT_EQ(ops.op.removeCurrent(l.mountKey("s2"), Retry::standard()), Removal::Removed); + const auto got = ops.op.read(l.mountKey("s2"), Retry::standard()); + MountLease farewell = decodeMountLease(got->bytes); + farewell.min_active_build_sequence = std::numeric_limits::max(); + mustCommit(ops.op.replace(l.mountKey("s2"), encodeMountLease(farewell), got->etag, Retry::standard()), "s2 farewell"); - computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(kStableThresholdMs), kStableThresholdMs, obs); + const HeartbeatFloor floor = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(kStableThresholdMs), kObserverLeaseMs, kCadenceMs, obs, warned); + EXPECT_EQ(floor.terminated, 1u); EXPECT_TRUE(obs.contains("s1")); - EXPECT_FALSE(obs.contains("s2")) - << "a srid removed from the LIST entirely must be pruned from obs, not linger forever"; + EXPECT_FALSE(obs.contains("s2")); +} + +/// A slot deleted by a 26.6.4 decommission or by hand is never listed again; its observation is dropped. +TEST(CASHeartbeatFloor, UnseenSridPrunedFromObservationMap) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + seedMount(ops.op, l, "s1", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); + seedMount(ops.op, l, "s2", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); + MountObservationMap obs; + std::set warned; + computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); + ASSERT_TRUE(obs.contains("s2")); + + /// s1 renews so it stays observed for its own reason; only s2's absence from the listing is under test. + renewMount(ops.op, l, "s1"); + ASSERT_EQ(ops.op.removeCurrent(l.mountKey("s2"), Retry::standard()), Removal::Removed); + computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(kStableThresholdMs), kObserverLeaseMs, kCadenceMs, obs, warned); + EXPECT_TRUE(obs.contains("s1")); + EXPECT_FALSE(obs.contains("s2")); } TEST(CASHeartbeatFloor, ClassifiesAndFencesOut) @@ -1670,9 +2158,10 @@ TEST(CASHeartbeatFloor, ClassifiesAndFencesOut) /*min_active_build_sequence*/ std::numeric_limits::max()); MountObservationMap obs; + std::set warned; /// Round 1 (mono 0): first sight of every non-terminal mount — nothing is fence-eligible yet. - const HeartbeatFloor floor_before = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kStableThresholdMs, obs); + const HeartbeatFloor floor_before = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(floor_before.live, 3u); // s1, s2, s3: observation just started EXPECT_EQ(floor_before.terminated, 1u); // s5 EXPECT_EQ(floor_before.fenced_now, 0u); @@ -1690,7 +2179,7 @@ TEST(CASHeartbeatFloor, ClassifiesAndFencesOut) /// Round 2 (mono == threshold): s1/s2's renewed incarnations restart their observation (still /// live); s3's original incarnation has now held stable for the full threshold -> fenced. const HeartbeatFloor floor2 = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(kStableThresholdMs), - kStableThresholdMs, obs); + kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(floor2.live, 2u); // s1, s2: renewed, observation restarted EXPECT_EQ(floor2.terminated, 1u); // s5 @@ -1765,9 +2254,10 @@ TEST(CASHeartbeatFloor, FenceOutLosesTheIncarnationRaceAndReclassifiesLive) seedMount(ops.op, l, "s1", /*expires*/ kNowMs - 60'000, /*fenced*/ false, /*min_active_build_sequence*/ 0); MountObservationMap obs; + std::set warned; /// Round 1: first sight, observation starts — never reaches the fence-out path (the race /// decorator stays armed for round 2). - const HeartbeatFloor floor_before = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kStableThresholdMs, obs); + const HeartbeatFloor floor_before = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(floor_before.fenced_now, 0u); /// Round 2: the incarnation has been stable past threshold, so the function attempts the @@ -1775,7 +2265,7 @@ TEST(CASHeartbeatFloor, FenceOutLosesTheIncarnationRaceAndReclassifiesLive) /// and the re-decision reclassifies the slot as live (observation restarted on the new /// incarnation) — never fenced. const HeartbeatFloor floor2 = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(kStableThresholdMs), - kStableThresholdMs, obs); + kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(floor2.fenced_now, 0u); EXPECT_EQ(floor2.live, 1u); @@ -1840,20 +2330,21 @@ TEST(CASHeartbeatFloor, ConfirmingCallDecidesOnItsEntrySample) seedMount(ops.op, l, "s1", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); const auto clock = [&mono] { return mono; }; MountObservationMap obs; + std::set warned; - computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs); + computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned); ASSERT_EQ(obs.at("s1").first_seen_mono_ms, 0u); /// The call starts one millisecond short of the threshold; its walk passes the threshold many times. mono = kStableThresholdMs - 1; b->read_cost_ms = 10 * kStableThresholdMs; - const HeartbeatFloor slow = computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs); + const HeartbeatFloor slow = computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(slow.fenced_now, 0u); EXPECT_EQ(slow.live, 1u); EXPECT_EQ(obs.at("s1").first_seen_mono_ms, 0u) << "an unchanged token keeps its first sighting"; b->read_cost_ms = 0; - EXPECT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs).fenced_now, 1u); + EXPECT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned).fenced_now, 1u); } TEST(CASHeartbeatFloor, FailedSlotReadKeepsTheWatch) @@ -1865,19 +2356,20 @@ TEST(CASHeartbeatFloor, FailedSlotReadKeepsTheWatch) seedMount(ops.op, l, "s1", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); const auto clock = [&mono] { return mono; }; MountObservationMap obs; + std::set warned; - computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs); + computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned); ASSERT_EQ(obs.at("s1").first_seen_mono_ms, 0u); mono = kStableThresholdMs / 2; b->fail_slot_reads = true; - EXPECT_ANY_THROW(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs)); + EXPECT_ANY_THROW(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned)); ASSERT_TRUE(obs.contains("s1")); EXPECT_EQ(obs.at("s1").first_seen_mono_ms, 0u); b->fail_slot_reads = false; mono = kStableThresholdMs; - EXPECT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs).fenced_now, 1u) + EXPECT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned).fenced_now, 1u) << "a failed read restarted the observation"; } @@ -1888,13 +2380,14 @@ TEST(CASHeartbeatFloor, SampleOlderThanTheSightingNeverFences) Ops ops(b); seedMount(ops.op, l, "s1", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); MountObservationMap obs; + std::set warned; - computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(kStableThresholdMs), kStableThresholdMs, obs); + computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(kStableThresholdMs), kObserverLeaseMs, kCadenceMs, obs, warned); ASSERT_EQ(obs.at("s1").first_seen_mono_ms, kStableThresholdMs); /// A clock that went back: the unsigned difference would wrap past any threshold. const HeartbeatFloor floor = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(kStableThresholdMs - 1), - kStableThresholdMs, obs); + kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(floor.fenced_now, 0u); EXPECT_EQ(floor.live, 1u); } @@ -1913,22 +2406,23 @@ TEST(CASHeartbeatFloor, FirstSightingCountsFromAfterItsRead) seedMount(ops.op, l, "s1", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); const auto clock = [&mono] { return mono; }; MountObservationMap obs; + std::set warned; /// The call starts at 0 and reaches the slot at `kWalkMs`. b->read_cost_ms = kWalkMs; - ASSERT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs).fenced_now, 0u); + ASSERT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned).fenced_now, 0u); ASSERT_TRUE(obs.contains("s1")); EXPECT_EQ(obs.at("s1").first_seen_mono_ms, kWalkMs); /// One threshold after the first call started, less than one after its read. b->read_cost_ms = 0; mono = kStableThresholdMs; - const HeartbeatFloor early = computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs); + const HeartbeatFloor early = computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(early.fenced_now, 0u) << "fenced a token watched for less than the threshold"; EXPECT_EQ(early.live, 1u); mono = kWalkMs + kStableThresholdMs; - EXPECT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs).fenced_now, 1u); + EXPECT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned).fenced_now, 1u); } TEST(CASHeartbeatFloor, RedecisionAfterARefusedFenceCountsFromItsRead) @@ -1940,8 +2434,9 @@ TEST(CASHeartbeatFloor, RedecisionAfterARefusedFenceCountsFromItsRead) seedMount(ops.op, l, "s1", /*expires*/ 10, /*fenced*/ false, /*min_active_build_sequence*/ 0); const auto clock = [&mono] { return mono; }; MountObservationMap obs; + std::set warned; - computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs); + computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned); ASSERT_TRUE(obs.contains("s1")); const Etag first = obs.at("s1").etag; @@ -1950,7 +2445,7 @@ TEST(CASHeartbeatFloor, RedecisionAfterARefusedFenceCountsFromItsRead) mono = kStableThresholdMs; b->read_cost_ms = kWalkMs; b->renew_before_next_fence = true; - const HeartbeatFloor refused = computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs); + const HeartbeatFloor refused = computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned); ASSERT_FALSE(b->renew_before_next_fence) << "the call never attempted the fence-out"; ASSERT_EQ(refused.fenced_now, 0u); ASSERT_EQ(refused.live, 1u); @@ -1964,11 +2459,11 @@ TEST(CASHeartbeatFloor, RedecisionAfterARefusedFenceCountsFromItsRead) /// One threshold after that call started, less than one after its re-read. b->read_cost_ms = 0; mono = 2 * kStableThresholdMs; - EXPECT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs).fenced_now, 0u) + EXPECT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned).fenced_now, 0u) << "fenced the renewed token before it was watched for the threshold"; mono = reread_at + kStableThresholdMs; - EXPECT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kStableThresholdMs, obs).fenced_now, 1u); + EXPECT_EQ(computeHeartbeatFloor(ops.op, l, kNowMs, clock, kObserverLeaseMs, kCadenceMs, obs, warned).fenced_now, 1u); } TEST(CASHeartbeatFloor, EmptyPrefixYieldsNoLiveMounts) @@ -1978,7 +2473,8 @@ TEST(CASHeartbeatFloor, EmptyPrefixYieldsNoLiveMounts) Ops ops(b); MountObservationMap obs; - const HeartbeatFloor floor = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kStableThresholdMs, obs); + std::set warned; + const HeartbeatFloor floor = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); EXPECT_EQ(floor.live, 0u); EXPECT_EQ(floor.terminated, 0u); @@ -1986,6 +2482,239 @@ TEST(CASHeartbeatFloor, EmptyPrefixYieldsNoLiveMounts) EXPECT_EQ(floor.already_fenced, 0u); } +namespace +{ +HeartbeatFloor floorAt(Ops & ops, const Layout & l, uint64_t mono_ms, MountObservationMap & obs, std::set & warned) +{ + return computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(mono_ms), kObserverLeaseMs, kCadenceMs, obs, warned); +} +} + +/// The body declares 60 s against the observer's 30 s: 60 s + 3 s + the 1 s step. +TEST(CASHeartbeatFloor, HolderDeclaringALongerLeaseIsWatchedForThatLease) +{ + Ops ops(std::make_shared()); + Layout l("p"); + seedMount(ops.op, l, "s1", /*expires*/ kNowMs + 60'000, /*fenced*/ false, /*min_active_build_sequence*/ 0); + MountObservationMap obs; + std::set warned; + floorAt(ops, l, 0, obs, warned); + EXPECT_EQ(floorAt(ops, l, kStableThresholdMs, obs, warned).fenced_now, 0u) + << "fenced after the observer's own lease while the declared lease still ran"; + EXPECT_EQ(floorAt(ops, l, 63'999, obs, warned).fenced_now, 0u); + EXPECT_EQ(floorAt(ops, l, 64'000, obs, warned).fenced_now, 1u); +} + +/// A 10 minute declared lease (the longest a new holder declares) against the observer's 30 s: 600 s + 30 s + the 1 s step. +TEST(CASHeartbeatFloor, HolderDeclaringTheMaximumLeaseIsFencedAtItsThresholdNotLater) +{ + Ops ops(std::make_shared()); + Layout l("p"); + seedMount(ops.op, l, "s1", /*expires*/ kNowMs + 600'000, /*fenced*/ false, /*min_active_build_sequence*/ 0); + MountObservationMap obs; + std::set warned; + floorAt(ops, l, 0, obs, warned); + EXPECT_EQ(floorAt(ops, l, 630'999, obs, warned).fenced_now, 0u); + EXPECT_EQ(floorAt(ops, l, 631'000, obs, warned).fenced_now, 1u); +} + +/// A shorter declared lease, a missing start stamp, equal stamps, an expiry before the start. +TEST(CASHeartbeatFloor, ShorterOrUnusableDeclaredLeaseIsWatchedForTheObserversOwn) +{ + Ops ops(std::make_shared()); + Layout l("p"); + seedMount(ops.op, l, "s1", /*expires*/ kNowMs + 10'000, false, 0); + seedMount(ops.op, l, "s2", /*expires*/ 60'000, false, 0, /*seq*/ 1, /*started_at_ms*/ 0); + seedMount(ops.op, l, "s3", /*expires*/ kNowMs, false, 0); + seedMount(ops.op, l, "s4", /*expires*/ kNowMs - 1, false, 0); + MountObservationMap obs; + std::set warned; + floorAt(ops, l, 0, obs, warned); + EXPECT_EQ(floorAt(ops, l, kStableThresholdMs - 1, obs, warned).fenced_now, 0u); + EXPECT_EQ(floorAt(ops, l, kStableThresholdMs, obs, warned).fenced_now, 4u); +} + +/// `gc_fenced` and farewell slots stay terminal whatever lease they declare: no observation, no write. +TEST(CASHeartbeatFloor, TerminalSlotsAreExcludedWhateverTheyDeclare) +{ + Ops ops(std::make_shared()); + Layout l("p"); + seedMount(ops.op, l, "s1", /*expires*/ kNowMs + 60'000, /*fenced*/ true, 0); + seedMount(ops.op, l, "s2", /*expires*/ kNowMs + 60'000, false, std::numeric_limits::max()); + MountObservationMap obs; + std::set warned; + const HeartbeatFloor floor = floorAt(ops, l, 0, obs, warned); + EXPECT_EQ(floor.already_fenced, 1u); + EXPECT_EQ(floor.terminated, 1u); + EXPECT_TRUE(obs.empty()); +} + +TEST(CASMountObservationThreshold, LongerOfOwnAndDeclaredLeasePlusDriftAndStep) +{ + EXPECT_EQ(mountObservationThresholdMs(30'000, 0, 1'000), 32'500u); + EXPECT_EQ(mountObservationThresholdMs(30'000, 10'000, 1'000), 32'500u); + EXPECT_EQ(mountObservationThresholdMs(30'000, 60'000, 1'000), 64'000u); +} + +TEST(CASMountObservationThreshold, SaturatesInsteadOfWrapping) +{ + constexpr uint64_t max = std::numeric_limits::max(); + /// `20 * (max / 21)` plus its twentieth is `max - 15`: the step alone decides the overflow. + EXPECT_EQ(mountObservationThresholdMs(0, 20 * (max / 21), 14), max - 1); + EXPECT_EQ(mountObservationThresholdMs(0, 20 * (max / 21), 16), max); + EXPECT_EQ(mountObservationThresholdMs(30'000, max - 10, 0), max); +} + +TEST(CASMountObservationThreshold, DeclaredLeaseIsThePositiveStampInterval) +{ + MountLease m; + m.started_at_ms = 1'000; + m.expires_at_ms = 61'000; + EXPECT_EQ(declaredMountLeaseMs(m), 60'000u); + m.started_at_ms = 0; + EXPECT_EQ(declaredMountLeaseMs(m), 0u) << "a body without the start stamp"; + m.started_at_ms = 61'000; + EXPECT_EQ(declaredMountLeaseMs(m), 0u) << "a farewell stamps both equal"; + m.started_at_ms = 70'000; + EXPECT_EQ(declaredMountLeaseMs(m), 0u) << "expiry before start"; +} + +TEST(CASHeartbeatFloor, HolderWithADifferentLeaseIsWarnedAboutWithoutChangingTheClassification) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + seedMount(ops.op, l, "s1", /*expires*/ kNowMs + 60'000, /*fenced*/ false, /*min_active_build_sequence*/ 0); + const String before = ops.op.read(l.mountKey("s1"), Retry::standard())->bytes; + + DB::Cas::tests::ScopedLogCapture capture("CasHeartbeatFloor", "warning"); + MountObservationMap obs; + std::set warned; + const HeartbeatFloor floor = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); + + EXPECT_EQ(capture.count( + "This server is configured with cas_mount_lease_ttl_ms=30000 but the holder of slot s1 " + "(hostname=h-s1, pid=100) declares a lease of 60000 ms. Observers wait for the longer of the two; " + "a longer lease on one server delays recovery after that server's crash."), 1u) + << capture.captured(); + EXPECT_EQ(floor.live, 1u); + EXPECT_EQ(floor.fenced_now, 0u); + EXPECT_EQ(ops.op.read(l.mountKey("s1"), Retry::standard())->bytes, before); + EXPECT_EQ(warned, std::set{"s1"}); +} + +TEST(CASHeartbeatFloor, HolderWithTheSameLeaseIsNotWarnedAbout) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + seedMount(ops.op, l, "s1", /*expires*/ kNowMs + kObserverLeaseMs, /*fenced*/ false, /*min_active_build_sequence*/ 0); + + DB::Cas::tests::ScopedLogCapture capture("CasHeartbeatFloor", "warning"); + MountObservationMap obs; + std::set warned; + computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); + + EXPECT_EQ(capture.count(kLeaseMismatchMarker), 0u) << capture.captured(); + EXPECT_TRUE(warned.empty()); +} + +TEST(CASHeartbeatFloor, FarewellBodyIsNotWarnedAbout) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + /// What `MountLeaseRenewer::terminate` writes: both stamps equal, farewell sentinel. + seedMount(ops.op, l, "s1", /*expires*/ kNowMs, /*fenced*/ false, + /*min_active_build_sequence*/ std::numeric_limits::max()); + + DB::Cas::tests::ScopedLogCapture capture("CasHeartbeatFloor", "warning"); + MountObservationMap obs; + std::set warned; + const HeartbeatFloor floor = computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); + + EXPECT_EQ(floor.terminated, 1u); + EXPECT_EQ(capture.count(kLeaseMismatchMarker), 0u) << capture.captured(); +} + +/// Both slots keep a 60 s body that differs from the observer's 30 s, but a fenced or departed holder +/// cannot delay anything, so naming it would only alarm the operator. +TEST(CASHeartbeatFloor, TerminalSlotsWithADifferentLeaseAreNotWarnedAbout) +{ + Ops ops(std::make_shared()); + Layout l("p"); + seedMount(ops.op, l, "s1", /*expires*/ kNowMs + 60'000, /*fenced*/ true, /*min_active_build_sequence*/ 0); + seedMount(ops.op, l, "s2", /*expires*/ kNowMs + 60'000, /*fenced*/ false, + /*min_active_build_sequence*/ std::numeric_limits::max()); + + DB::Cas::tests::ScopedLogCapture capture("CasHeartbeatFloor", "warning"); + MountObservationMap obs; + std::set warned; + const HeartbeatFloor floor = floorAt(ops, l, 0, obs, warned); + + EXPECT_EQ(floor.already_fenced, 1u); + EXPECT_EQ(floor.terminated, 1u); + EXPECT_EQ(capture.count(kLeaseMismatchMarker), 0u) << capture.captured(); + EXPECT_TRUE(warned.empty()); +} + +TEST(CASHeartbeatFloor, MissingStartStampIsNotWarnedAbout) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + /// A body without `started_at_ms` decodes it as 0; the interval to `expires_at_ms` means nothing. + seedMount(ops.op, l, "s1", /*expires*/ 60'000, /*fenced*/ false, /*min_active_build_sequence*/ 0, + /*seq*/ 1, /*started_at_ms*/ 0); + + DB::Cas::tests::ScopedLogCapture capture("CasHeartbeatFloor", "warning"); + MountObservationMap obs; + std::set warned; + computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); + + EXPECT_EQ(capture.count(kLeaseMismatchMarker), 0u) << capture.captured(); +} + +TEST(CASHeartbeatFloor, LeaseMismatchIsWarnedOncePerServerRootId) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + seedMount(ops.op, l, "s1", /*expires*/ kNowMs + 60'000, /*fenced*/ false, /*min_active_build_sequence*/ 0); + seedMount(ops.op, l, "s2", /*expires*/ kNowMs + 60'000, /*fenced*/ false, /*min_active_build_sequence*/ 0); + + DB::Cas::tests::ScopedLogCapture capture("CasHeartbeatFloor", "warning"); + MountObservationMap obs; + std::set warned; + computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); + /// A renewal gives s1 a new incarnation; the next round reads both bodies again. + renewMount(ops.op, l, "s1"); + computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(1), kObserverLeaseMs, kCadenceMs, obs, warned); + + EXPECT_EQ(capture.count(kLeaseMismatchMarker), 2u) << capture.captured(); + EXPECT_EQ(capture.count("holder of slot s1 ("), 1u) << capture.captured(); + EXPECT_EQ(capture.count("holder of slot s2 ("), 1u) << capture.captured(); +} + +TEST(CASHeartbeatFloor, UnseenSridPrunedFromLeaseMismatchWarned) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + seedMount(ops.op, l, "s1", /*expires*/ kNowMs + 60'000, /*fenced*/ false, /*min_active_build_sequence*/ 0); + seedMount(ops.op, l, "s2", /*expires*/ kNowMs + 60'000, /*fenced*/ false, /*min_active_build_sequence*/ 0); + + MountObservationMap obs; + std::set warned; + computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(0), kObserverLeaseMs, kCadenceMs, obs, warned); + ASSERT_EQ(warned, (std::set{"s1", "s2"})); + + ASSERT_EQ(ops.op.removeCurrent(l.mountKey("s2"), Retry::standard()), Removal::Removed); + computeHeartbeatFloor(ops.op, l, kNowMs, frozenClock(1), kObserverLeaseMs, kCadenceMs, obs, warned); + + EXPECT_EQ(warned, std::set{"s1"}); +} + /// ---- Task 1 (Phase 2): `listMounts` — read-only mount-slot enumeration for introspection ---- TEST(CASListMounts, ClassifiesEveryStateReadOnly) @@ -2025,9 +2754,9 @@ TEST(CASListMounts, ClassifiesEveryStateReadOnly) } } -/// A `srid` may itself contain `/` (e.g. `shard-01/replica-a` — legal per -/// `CASServerRootId.ValidationAcceptsCleanPathsRejectsBad`). Slicing the key by the last `/` before -/// the `/mount` suffix (as opposed to by `serverRootsPrefix()` length) truncates it to `replica-a`. +/// A pool may hold a mount slot whose `srid` contains `/`, written before `validateServerRootId` refused +/// one; `system.cas_mounts` must still name it whole. Slicing the key by the last `/` before the `/mount` +/// suffix (as opposed to by `serverRootsPrefix()` length) truncates it to `replica-a`. TEST(CASListMounts, NestedSridIsNotTruncated) { auto backend = std::make_shared(); @@ -2075,6 +2804,151 @@ TEST(CASClaimMount, SameEpochFencedIsNotRefreshable) MountClaimResult::Claimed); } +/// An older allocation that reaches the slot late must not replace a newer incarnation of this server, +/// whatever certifies that incarnation's end: recovery seals only epochs below the live one. +TEST(CASClaimMountMonotonicity, LowerEpochOverANewerBodyFailsWithoutWriting) +{ + enum class Shape { Live, GcFenced, CleanFarewell, ProvenDead, UnsafeAuthorized }; + for (const Shape shape : {Shape::Live, Shape::GcFenced, Shape::CleanFarewell, Shape::ProvenDead, + Shape::UnsafeAuthorized}) + { + SCOPED_TRACE(static_cast(shape)); + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + const String key = l.mountKey("r"); + ASSERT_EQ(claimMount(ops.op, l, "r", UInt128(1), /*our_epoch=*/8, 1000, 30000).kind, MountClaimResult::Claimed); + if (shape == Shape::GcFenced || shape == Shape::CleanFarewell) + { + const auto got = ops.op.read(key, Retry::standard()); + ASSERT_TRUE(got.has_value()); + MountLease body = decodeMountLease(got->bytes); + ++body.seq; + if (shape == Shape::GcFenced) + body.gc_fenced = true; + else + body.min_active_build_sequence = std::numeric_limits::max(); + mustCommit(ops.op.replace(key, encodeMountLease(body), got->etag, Retry::standard()), "terminal newer body"); + } + const auto before = ops.op.read(key, Retry::standard()); + ASSERT_TRUE(before.has_value()); + const std::optional token = before->etag; + + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::ABORTED, "newer than the writer_epoch", [&] + { + (void)claimMount(ops.op, l, "r", UInt128(1), /*our_epoch=*/7, 2000, 30000, + shape == Shape::ProvenDead ? token : std::nullopt, /*sink=*/{}, + shape == Shape::UnsafeAuthorized ? token : std::nullopt); + }); + EXPECT_EQ(ops.op.read(key, Retry::standard())->bytes, before->bytes) << "the newer body was replaced"; + } +} + +/// Two processes holding one epoch, as 26.6.4 allocators without the counter nonce could produce. +TEST(CASClaimMountMonotonicity, TwoClaimersOfOneEpochDoNotShareTheSlot) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + const uint64_t shared_epoch = 2; + + /// The competitor claims first. + ASSERT_EQ(claimMount(ops.op, l, "r", UInt128(1), shared_epoch, 1000, 30000).kind, MountClaimResult::Claimed); + const auto competitor = ops.op.read(l.mountKey("r"), Retry::standard()); + ASSERT_TRUE(competitor.has_value()); + + const MountClaimResult ours = claimMount(ops.op, l, "r", UInt128(1), shared_epoch, 2000, 30000); + EXPECT_EQ(ours.kind, MountClaimResult::LiveDoubleStart); + ASSERT_TRUE(ours.body.has_value()); + EXPECT_EQ(ours.body->write_attempt_id, decodeMountLease(competitor->bytes).write_attempt_id); + ASSERT_TRUE(ours.etag.has_value()); + EXPECT_TRUE(*ours.etag == competitor->etag); + EXPECT_EQ(ops.op.read(l.mountKey("r"), Retry::standard())->bytes, competitor->bytes) << "the slot was written"; +} + +TEST(CASClaimMountMonotonicity, AwaitingExpiryReturnsAtOnceOnABodyOfOurOwnPair) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + /// Another process allocated epoch 7 as well; its body is in the slot. + ASSERT_EQ(claimMount(ops.op, l, "r", UInt128(1), 7, 1000, 100).kind, MountClaimResult::Claimed); + const auto before = ops.op.read(l.mountKey("r"), Retry::standard()); + ASSERT_TRUE(before.has_value()); + + uint64_t mono = 0; + int sleeps = 0; + const auto r = claimMountAwaitingExpiry(ops.op, l, "r", UInt128(1), /*our_epoch=*/7, + [] { return uint64_t{1000}; }, [&] { return mono; }, /*ttl_ms=*/100, /*poll_interval_ms=*/20, + [&](uint64_t ms) + { + mono += ms; + if (++sleeps > 100) + throw std::runtime_error("the observation of our own (uuid, epoch) never ended"); + }); + EXPECT_EQ(r.kind, MountClaimResult::LiveDoubleStart); + EXPECT_EQ(sleeps, 0) << "no observation can authorize replacing our own pair"; + EXPECT_EQ(ops.op.read(l.mountKey("r"), Retry::standard())->bytes, before->bytes); +} + +TEST(CASClaimMountMonotonicity, AwaitingExpiryFailsAtOnceWhenARacedWriteMeetsANewerEpoch) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + /// Absent at our read; a newer start of this server mints the slot before our create lands. + b->before_put_if_absent = [&] + { + DB::Cas::tests::OperationForTest racer(*b); + ASSERT_EQ(claimMount(*racer, l, "r", UInt128(1), /*our_epoch=*/8, 1000, 100).kind, MountClaimResult::Claimed); + }; + + uint64_t mono = 0; + int sleeps = 0; + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::ABORTED, "newer than the writer_epoch", [&] + { + (void)claimMountAwaitingExpiry(ops.op, l, "r", UInt128(1), /*our_epoch=*/7, + [] { return uint64_t{1000}; }, [&] { return mono; }, /*ttl_ms=*/100, /*poll_interval_ms=*/20, + [&](uint64_t ms) + { + mono += ms; + ++sleeps; + }); + }); + EXPECT_FALSE(b->before_put_if_absent) << "the newer start was never installed"; + EXPECT_EQ(sleeps, 0) << "a newer incarnation is refused, not observed"; + EXPECT_EQ(decodeMountLease(ops.op.read(l.mountKey("r"), Retry::standard())->bytes).writer_epoch, 8u); +} + +/// The engine settles our own landed attempt inside the write call, so a claim never meets its own body +/// on a later call. +TEST(CASClaimMountMonotonicity, AConfirmedLandedReclaimIsStillClaimed) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + b->key = l.mountKey("r"); + ASSERT_EQ(claimMount(ops.op, l, "r", UInt128(1), 7, 1000, 30000).kind, MountClaimResult::Claimed); + { + const auto got = ops.op.read(b->key, Retry::standard()); + ASSERT_TRUE(got.has_value()); + MountLease fenced = decodeMountLease(got->bytes); + fenced.gc_fenced = true; + ++fenced.seq; + mustCommit(ops.op.replace(b->key, encodeMountLease(fenced), got->etag, Retry::standard()), "fence-out"); + } + + b->armed = true; + const MountClaimResult r = claimMount(ops.op, l, "r", UInt128(1), 8, 2000, 30000); + ASSERT_FALSE(b->armed) << "the lost response was never injected"; + EXPECT_EQ(r.kind, MountClaimResult::Claimed); + EXPECT_EQ(r.prior, MountPriorState::Fenced); + ASSERT_TRUE(r.body.has_value()); + const MountLease slot = decodeMountLease(ops.op.read(b->key, Retry::standard())->bytes); + EXPECT_EQ(slot.writer_epoch, 8u); + EXPECT_EQ(slot.write_attempt_id, r.body->write_attempt_id); +} + /// ---- rev.6 Task 4: observation-based lease reclaim (no cross-node wall-clock trust) ---- /// A same-uuid, different-epoch lease whose STAMPED `expires_at_ms` looks long expired on OUR wall @@ -2291,7 +3165,7 @@ TEST(CASServerRoot, AllocateWriterEpochKeepsThePostConflictCorruptionCheck) auto backend = std::make_shared(/*reveal_owned_work=*/true); Ops ops(backend); EXPECT_THROW(allocateWriterEpoch( - ops.op, layout, "root/x", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), DB::Exception); + ops.op, layout, "root/x", emptyCatalogObservation()), DB::Exception); EXPECT_TRUE(backend->fired); ASSERT_TRUE(backend->winner_installed); } @@ -2299,7 +3173,7 @@ TEST(CASServerRoot, AllocateWriterEpochKeepsThePostConflictCorruptionCheck) auto backend = std::make_shared(/*reveal_owned_work=*/false); Ops ops(backend); EXPECT_EQ(allocateWriterEpoch( - ops.op, layout, "root/x", EpochMintPolicy::NormalMount, 0, emptyCatalogObservation()), 2u) + ops.op, layout, "root/x", emptyCatalogObservation()), 2u) << "the second decision must allocate from the conflict winner's epoch, not refuse outright"; EXPECT_TRUE(backend->fired); ASSERT_TRUE(backend->winner_installed); @@ -2451,6 +3325,35 @@ TEST(CASServerRootClaim, OwnerLostToARacerIsDecidedFromTheConflictObservation) } } +/// A tombstone sent against the owner's first ETag, delivered after a tombstone and a rejoin. Without a +/// life the rejoined owner's bytes equal the first owner's, so a content-derived ETag would admit it. +TEST(CASOwner, DelayedTombstoneAfterRejoinFailsOnLife) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + const String key = l.ownerKey("r"); + const OwnerObject first{.server_uuid = UInt128(1), .retired_at_ms = std::nullopt, .owner_life = 1}; + mustCommit(ops.op.create(key, encodeOwner(first), Retry::standard()), "first owner"); + const Etag stale = currentEtag(ops.op, key); + + OwnerObject tombstone = first; + tombstone.retired_at_ms = 1000; + tombstone.owner_life = 2; + mustCommit(ops.op.replace(key, encodeOwner(tombstone), currentEtag(ops.op, key), Retry::standard()), "tombstone"); + OwnerObject rejoined = first; + rejoined.owner_life = 3; + mustCommit(ops.op.replace(key, encodeOwner(rejoined), currentEtag(ops.op, key), Retry::standard()), "rejoin"); + ASSERT_NE(currentEtag(ops.op, key), stale) << "the life makes the rejoined bytes differ from the first"; + + OwnerObject late = first; + late.retired_at_ms = 2000; + late.owner_life = 2; + const WriteResult landed = ops.op.replace(key, encodeOwner(late), stale, Retry::standard()); + EXPECT_TRUE(std::holds_alternative(landed)); + EXPECT_EQ(decodeOwner(ops.op.read(key, Retry::standard())->bytes).owner_life, 3u); +} + /// A remount re-anchors its lease BEFORE it arms the fence for the new incarnation, so the fence is /// still latched lost at that moment. The steady-state renewal is refused there — the sibling test /// above pins that — and the remount's own renewal has to be admitted off the fence, or the pool could @@ -2483,3 +3386,518 @@ TEST(CASMountLease, RemountRenewalIsAdmittedOffTheMountFence) CasOperation reader = open_requests.admit(); EXPECT_EQ(decodeMountLease(reader.read(l.mountKey("r"), Retry::standard())->bytes).seq, 2u); } + +namespace +{ +/// A rejoin by hand: the next life under `uuid`, not retired. +void rewriteOwnerAt(Backend & backend, const Layout & l, const String & srid, UInt128 uuid) +{ + DB::Cas::tests::OperationForTest op(backend); + const auto current = (*op).read(l.ownerKey(srid), Retry::once()); + ASSERT_TRUE(current.has_value()); + const OwnerObject next{ + .server_uuid = uuid, .retired_at_ms = std::nullopt, .owner_life = decodeOwner(current->bytes).owner_life + 1, + .retirement_verified = false}; + ASSERT_TRUE(std::holds_alternative( + (*op).replace(l.ownerKey(srid), encodeOwner(next), current->etag, Retry::once()))); +} + +MountLease slotBody(Backend & backend, const Layout & l, const String & srid) +{ + DB::Cas::tests::OperationForTest op(backend); + const auto current = (*op).read(l.mountKey(srid), Retry::once()); + if (!current) + throw std::runtime_error("mount absent: " + l.mountKey(srid)); + return decodeMountLease(current->bytes); +} + +PoolConfig rootConfig() +{ + return PoolConfig{.pool_prefix = "p", .server_id = UInt128(1), .server_root_id = "r"}; +} +} + +/// The tombstone lands between the owner claim and the mount write; the only write of the mount key in +/// this open is the reclaim of the first open's farewell. +TEST(CASMount, NormalMountRefusesARetiredOwnerAfterClaim) +{ + auto b = std::make_shared(); + Layout l("p"); + { auto first = Pool::open(b, rootConfig()); } + bool fired = false; + b->onBeforeWrite(l.mountKey("r"), [&] + { + if (std::exchange(fired, true)) + return; + DB::Cas::tests::tombstoneOwnerForTest(*b, l.ownerKey("r")); + }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "decommissioned", [&] { Pool::open(b, rootConfig()); }); + ASSERT_TRUE(fired); + const MountLease slot = slotBody(*b, l, "r"); + EXPECT_EQ(slot.server_uuid, UInt128(1)); + EXPECT_EQ(slot.writer_epoch, 2u) << "the refused claim's own body, not the first open's farewell"; + EXPECT_TRUE(slot.gc_fenced) << "the refused claimer fences itself so nobody waits its lease out"; +} + +/// A victim prechecked before a decommission wakes after a new-uuid rejoin and reclaims its own farewell. +TEST(CASMount, ClaimRecheckRefusesAForeignOwner) +{ + auto b = std::make_shared(); + Layout l("p"); + { auto first = Pool::open(b, rootConfig()); } + bool fired = false; + b->onBeforeWrite(l.mountKey("r"), [&] + { + if (std::exchange(fired, true)) + return; + rewriteOwnerAt(*b, l, "r", UInt128(2)); + }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "different server", [&] { Pool::open(b, rootConfig()); }); + ASSERT_TRUE(fired); + const MountLease slot = slotBody(*b, l, "r"); + EXPECT_EQ(slot.server_uuid, UInt128(1)); + EXPECT_EQ(slot.writer_epoch, 2u); + EXPECT_TRUE(slot.gc_fenced); +} + +TEST(CASMount, ClaimRecheckRefusesAnAbsentOwner) +{ + auto b = std::make_shared(); + Layout l("p"); + { auto first = Pool::open(b, rootConfig()); } + bool fired = false; + b->onBeforeWrite(l.mountKey("r"), [&] + { + if (std::exchange(fired, true)) + return; + DB::Cas::tests::OperationForTest op(*b); + ASSERT_EQ((*op).removeCurrent(l.ownerKey("r"), Retry::once()), Removal::Removed); + }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "vanished", [&] { Pool::open(b, rootConfig()); }); + ASSERT_TRUE(fired); + EXPECT_TRUE(slotBody(*b, l, "r").gc_fenced); +} + +/// The owner read refuses, and the self-fence write cannot reach the store: the refusal is still the +/// owner's reason and the slot keeps the claimer's live body for GC to fence. +TEST(CASMount, RecheckRefusalSurvivesAFailedSelfFence) +{ + auto b = std::make_shared(); + Layout l("p"); + { auto first = Pool::open(b, rootConfig()); } + bool fired = false; + b->onBeforeWrite(l.mountKey("r"), [&] + { + if (std::exchange(fired, true)) + return; + DB::Cas::tests::tombstoneOwnerForTest(*b, l.ownerKey("r")); + /// The claim write has already passed its armed-failure check, so this arming fails the next + /// mount write: the self-fence. + b->failNextWriteWith(l.mountKey("r"), std::make_exception_ptr( + DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "store refused the fence write"))); + }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "decommissioned", [&] { Pool::open(b, rootConfig()); }); + ASSERT_TRUE(fired); + const MountLease slot = slotBody(*b, l, "r"); + EXPECT_EQ(slot.writer_epoch, 2u) << "the claim itself landed"; + EXPECT_FALSE(slot.gc_fenced) << "the fence write was refused; the body is untouched"; +} + +/// The GC fenced our lease; the remount reclaims it, and the owner was tombstoned in between. +TEST(CASMount, FenceRecoveryReclaimRechecksTheOwner) +{ + auto b = std::make_shared(); + Layout l("p"); + auto store = Pool::open(b, rootConfig()); + DB::Cas::tests::fenceOutMountForTest(*b, l.mountKey("r")); + DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::ABORTED, [&] { store->renewWatermarkOnce(); }); + bool fired = false; + b->onBeforeWrite(l.mountKey("r"), [&] + { + if (std::exchange(fired, true)) + return; + DB::Cas::tests::tombstoneOwnerForTest(*b, l.ownerKey("r")); + }); + EXPECT_FALSE(store->tryRemountOnce()); + ASSERT_TRUE(fired); + const MountLease slot = slotBody(*b, l, "r"); + EXPECT_EQ(slot.writer_epoch, 2u) << "the remount's fresh epoch reached the slot"; + EXPECT_TRUE(slot.gc_fenced) << "and was fenced by the refused recheck"; + EXPECT_FALSE(store->mayMutate()); + + /// The next attempt reads the tombstone before it claims. + EXPECT_FALSE(store->tryRemountOnce()); + EXPECT_EQ(store->lifecycle(), PoolLifecycle::IdentityLost); + EXPECT_EQ(slotBody(*b, l, "r").writer_epoch, 2u); +} + +namespace +{ +void writeOwnerAt(Backend & backend, const Layout & l, const String & srid, const OwnerObject & owner) +{ + DB::Cas::tests::OperationForTest op(backend); + const auto current = (*op).read(l.ownerKey(srid), Retry::once()); + ASSERT_TRUE(current.has_value()); + ASSERT_TRUE(std::holds_alternative( + (*op).replace(l.ownerKey(srid), encodeOwner(owner), current->etag, Retry::once()))); +} + +/// The owner changed while `store` was fenced. The remount stops in `IdentityLost` before it claims the slot or +/// allocates an epoch, says `why` in its result, and counts one `CASIdentityLost`. +void expectRemountStopsForGood(const std::shared_ptr & b, const PoolPtr & store, + const std::function & change_owner, std::string_view why) +{ + const Layout l("p"); + change_owner(); + DB::Cas::tests::fenceOutMountForTest(*b, l.mountKey("r")); + DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::ABORTED, [&] { store->renewWatermarkOnce(); }); + Ops ops(b); + const String owner_before = ops.op.read(l.ownerKey("r"), Retry::standard())->bytes; + const String counter_before = ops.op.read(l.epochKey("r"), Retry::standard())->bytes; + const uint64_t epoch_before = slotBody(*b, l, "r").writer_epoch; + const uint64_t identity_lost_before = ProfileEvents::global_counters[ProfileEvents::CASIdentityLost]; + DB::Cas::tests::ScopedLoggerCapture capture("CasPool", "warning"); + + EXPECT_FALSE(store->tryRemountOnce()); + EXPECT_EQ(store->lifecycle(), PoolLifecycle::IdentityLost); + EXPECT_FALSE(store->tryRemountOnce()); + + EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASIdentityLost], identity_lost_before + 1); + EXPECT_NE(capture.captured().find(why), String::npos) << capture.captured(); + EXPECT_EQ(ops.op.read(l.epochKey("r"), Retry::standard())->bytes, counter_before) << "no epoch was allocated"; + EXPECT_EQ(ops.op.read(l.ownerKey("r"), Retry::standard())->bytes, owner_before) << "the owner was not written"; + const MountLease slot = slotBody(*b, l, "r"); + EXPECT_EQ(slot.writer_epoch, epoch_before); + EXPECT_TRUE(slot.gc_fenced); +} +} + +/// Another process of the same uuid rejoined (the life moved) while this one was fenced. +TEST(CASMount, RemountRefusesAfterASameUuidRejoin) +{ + auto b = std::make_shared(); + Layout l("p"); + auto store = Pool::open(b, rootConfig()); + expectRemountStopsForGood(b, store, [&] { rewriteOwnerAt(*b, l, "r", UInt128(1)); }, "moved from life 1 to life 2"); +} + +/// A member paused past its lease, fenced by GC, then decommissioned. +TEST(CASMount, RemountStopsForGoodOverARetiredOwner) +{ + auto b = std::make_shared(); + Layout l("p"); + auto store = Pool::open(b, rootConfig()); + expectRemountStopsForGood( + b, store, [&] { DB::Cas::tests::tombstoneOwnerForTest(*b, l.ownerKey("r")); }, + "was decommissioned while this process was fenced"); +} + +/// A member started over a 26.6.4 owner (no `owner_life`, so life 0) and a 26.6.4 decommission tombstoned it: the +/// tombstone also reads as life 0, so only the retirement itself ends the remount. +TEST(CASMount, RemountStopsForGoodOverA2664TombstoneAtLifeZero) +{ + auto b = std::make_shared(); + Layout l("p"); + { auto first = Pool::open(b, rootConfig()); } + writeOwnerAt(*b, l, "r", + OwnerObject{.server_uuid = UInt128(1), .retired_at_ms = std::nullopt, .owner_life = 0, .retirement_verified = false}); + auto store = Pool::open(b, rootConfig()); /// records life 0: a matching claim does not rewrite the owner + expectRemountStopsForGood( + b, store, + [&] + { + writeOwnerAt(*b, l, "r", + OwnerObject{.server_uuid = UInt128(1), .retired_at_ms = 1, .owner_life = 0, .retirement_verified = false}); + }, + "was decommissioned while this process was fenced"); +} + +/// Another server owns the id now; no retry can resume it. +TEST(CASMount, RemountStopsForGoodOverAForeignOwner) +{ + auto b = std::make_shared(); + Layout l("p"); + auto store = Pool::open(b, rootConfig()); + expectRemountStopsForGood(b, store, [&] { rewriteOwnerAt(*b, l, "r", UInt128(2)); }, "owned by a different server"); +} + +namespace +{ +void seedOwner(Ops & ops, const Layout & l, const OwnerObject & owner) +{ + mustCommit(ops.op.create(l.ownerKey("r"), encodeOwner(owner), Retry::standard()), "owner"); +} + +MountLease liveBody(UInt128 uuid, uint64_t epoch) +{ + return MountLease{ + .server_uuid = uuid, .writer_epoch = epoch, .hostname = "h", .pid = 1, .started_at_ms = 1, .seq = 1, + .expires_at_ms = 100, .min_active_build_sequence = 0, .gc_fenced = false, .write_attempt_id = UInt128(9)}; +} +} + +/// A refusing recheck rewrites only its own live `(uuid, epoch)` body; every other slot body keeps its bytes. +TEST(CASMount, RecheckSelfFenceLeavesAnyOtherBodyAlone) +{ + MountLease other_epoch = liveBody(UInt128(1), 6); + MountLease foreign = liveBody(UInt128(2), 7); + MountLease farewell = liveBody(UInt128(1), 7); + farewell.min_active_build_sequence = std::numeric_limits::max(); + MountLease fenced = liveBody(UInt128(1), 7); + fenced.gc_fenced = true; + const std::vector> cases{ + {"same uuid, another epoch", other_epoch}, {"foreign uuid", foreign}, {"clean farewell", farewell}, + {"already fenced", fenced}}; + for (const auto & [name, body] : cases) + { + SCOPED_TRACE(name); + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + seedOwner(ops, l, OwnerObject{.server_uuid = UInt128(1), .retired_at_ms = std::nullopt, .owner_life = 2, + .retirement_verified = false}); + mustCommit(ops.op.create(l.mountKey("r"), encodeMountLease(body), Retry::standard()), "slot"); + const auto before = ops.op.read(l.mountKey("r"), Retry::standard()); + ASSERT_TRUE(before.has_value()); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "owner moved from life 1 to life 2", [&] + { + recheckOwnerAfterClaim(ops.op, l, "r", + OwnerExpectation{.server_uuid = UInt128(1), .owner_life = 1, .retired = RetiredOwnerPolicy::Refuse}, 7); + }); + const auto after = ops.op.read(l.mountKey("r"), Retry::standard()); + ASSERT_TRUE(after.has_value()); + EXPECT_EQ(after->bytes, before->bytes); + EXPECT_EQ(after->etag, before->etag); + } +} + +/// The decommission admin accepts a retired owner, but not a foreign one or one at another life. +TEST(CASMount, RecheckForDecommissionStillRefusesForeignOwnerAndMovedLife) +{ + const OwnerExpectation admin{.server_uuid = UInt128(1), .owner_life = 2, .retired = RetiredOwnerPolicy::AcceptForDecommission}; + struct Case + { + String name; + OwnerObject owner; + String refusal; + }; + const std::vector cases{ + {"retired at the expected life", {.server_uuid = UInt128(1), .retired_at_ms = 5, .owner_life = 2, .retirement_verified = false}, ""}, + {"foreign uuid", {.server_uuid = UInt128(2), .retired_at_ms = 5, .owner_life = 2, .retirement_verified = false}, "different server"}, + {"moved life", {.server_uuid = UInt128(1), .retired_at_ms = 5, .owner_life = 3, .retirement_verified = false}, "owner moved from life 2 to life 3"}}; + for (const Case & c : cases) + { + SCOPED_TRACE(c.name); + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + seedOwner(ops, l, c.owner); + mustCommit(ops.op.create(l.mountKey("r"), encodeMountLease(liveBody(UInt128(1), 7)), Retry::standard()), "slot"); + const auto run = [&] { recheckOwnerAfterClaim(ops.op, l, "r", admin, 7); }; + if (c.refusal.empty()) + { + EXPECT_NO_THROW(run()); + EXPECT_FALSE(decodeMountLease(ops.op.read(l.mountKey("r"), Retry::standard())->bytes).gc_fenced); + } + else + { + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, c.refusal, run); + EXPECT_TRUE(decodeMountLease(ops.op.read(l.mountKey("r"), Retry::standard())->bytes).gc_fenced); + } + } +} + +namespace +{ +MountLease terminalBody(UInt128 uuid, uint64_t epoch, bool fenced) +{ + MountLease body = liveBody(uuid, epoch); + if (fenced) + body.gc_fenced = true; + else + body.min_active_build_sequence = std::numeric_limits::max(); + return body; +} + +const OwnerExpectation uuid_one_at_life_two{.server_uuid = UInt128(1), .owner_life = 2, .retired = RetiredOwnerPolicy::Refuse}; + +OwnerObject liveOwner(UInt128 uuid, uint64_t life) +{ + return OwnerObject{.server_uuid = uuid, .retired_at_ms = std::nullopt, .owner_life = life, .retirement_verified = false}; +} + +/// The slot's body with another `server_uuid`, everything else kept: what a rejoin under a new uuid leaves. +void rewriteSlotUuid(Backend & backend, const Layout & l, const String & srid, UInt128 uuid) +{ + DB::Cas::tests::OperationForTest op(backend); + const auto got = (*op).read(l.mountKey(srid), Retry::standard()); + ASSERT_TRUE(got.has_value()); + MountLease m = decodeMountLease(got->bytes); + m.server_uuid = uuid; + m.seq += 1; + ASSERT_TRUE(std::holds_alternative( + (*op).replace(l.mountKey(srid), encodeMountLease(m), got->etag, Retry::standard()))); +} +} + +/// After a rejoin under a new uuid the slot can hold another uuid's farewell or self-fence; the server +/// whose owner life it is takes it over. +TEST(CASMount, ForeignTerminalBodyIsReclaimable) +{ + for (const bool fenced : {false, true}) + { + SCOPED_TRACE(fenced ? "gc_fenced" : "clean farewell"); + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + seedOwner(ops, l, liveOwner(UInt128(1), 2)); + mustCommit(ops.op.create(l.mountKey("r"), encodeMountLease(terminalBody(UInt128(2), 7, fenced)), Retry::standard()), "slot"); + const MountClaimResult claim = claimMount(ops.op, l, "r", UInt128(1), 8, 1000, 100, {}, {}, {}, uuid_one_at_life_two); + ASSERT_EQ(claim.kind, MountClaimResult::Claimed); + EXPECT_EQ(claim.prior, fenced ? MountPriorState::Fenced : MountPriorState::Clean); + const MountLease slot = slotBody(*b, l, "r"); + EXPECT_EQ(slot.server_uuid, UInt128(1)); + EXPECT_EQ(slot.writer_epoch, 8u); + } +} + +/// The owner is read before the reclaim: a reclaim that a later recheck refuses would leave an old-uuid +/// body that a 26.6.4 starter (no recheck) could take. +TEST(CASMount, ForeignReclaimReadsTheOwnerFirst) +{ + struct Case + { + String name; + std::optional owner; + std::optional expected; + }; + OwnerObject retired = liveOwner(UInt128(1), 2); + retired.retired_at_ms = 5; + const std::vector cases{ + {"owner of another uuid", liveOwner(UInt128(3), 2), uuid_one_at_life_two}, + {"owner retired", retired, uuid_one_at_life_two}, + {"owner at another life", liveOwner(UInt128(1), 3), uuid_one_at_life_two}, + {"owner absent", std::nullopt, uuid_one_at_life_two}, + {"no expectation passed", liveOwner(UInt128(1), 2), std::nullopt}}; + for (const Case & c : cases) + { + SCOPED_TRACE(c.name); + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + if (c.owner) + seedOwner(ops, l, *c.owner); + mustCommit(ops.op.create(l.mountKey("r"), encodeMountLease(terminalBody(UInt128(2), 7, false)), Retry::standard()), "slot"); + const auto before = ops.op.read(l.mountKey("r"), Retry::standard()); + EXPECT_EQ(claimMount(ops.op, l, "r", UInt128(1), 8, 1000, 100, {}, {}, {}, c.expected).kind, MountClaimResult::ForeignOwner); + EXPECT_EQ(ops.op.read(l.mountKey("r"), Retry::standard())->etag, before->etag); + } +} + +/// A foreign body at or above our epoch is never replaced: a delayed claimer with a lower epoch stays out. +TEST(CASMount, ForeignTerminalBodyAtOrAboveOurEpochStaysForeign) +{ + for (const uint64_t body_epoch : {8, 9}) + { + SCOPED_TRACE(body_epoch); + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + seedOwner(ops, l, liveOwner(UInt128(1), 2)); + mustCommit(ops.op.create(l.mountKey("r"), encodeMountLease(terminalBody(UInt128(2), body_epoch, false)), Retry::standard()), "slot"); + const auto before = ops.op.read(l.mountKey("r"), Retry::standard()); + EXPECT_EQ(claimMount(ops.op, l, "r", UInt128(1), 8, 1000, 100, {}, {}, {}, uuid_one_at_life_two).kind, MountClaimResult::ForeignOwner); + EXPECT_EQ(ops.op.read(l.mountKey("r"), Retry::standard())->etag, before->etag); + } +} + +/// A live body of another uuid is never taken, whatever the owner says. +TEST(CASMount, ForeignLiveBodyStaysForeignOwner) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + seedOwner(ops, l, liveOwner(UInt128(1), 2)); + mustCommit(ops.op.create(l.mountKey("r"), encodeMountLease(liveBody(UInt128(2), 7)), Retry::standard()), "slot"); + const auto before = ops.op.read(l.mountKey("r"), Retry::standard()); + EXPECT_EQ(claimMount(ops.op, l, "r", UInt128(1), 8, 1000, 100, {}, {}, {}, uuid_one_at_life_two).kind, MountClaimResult::ForeignOwner); + EXPECT_EQ(ops.op.read(l.mountKey("r"), Retry::standard())->etag, before->etag); +} + +/// A start and a self-remount pass their owner expectation, so both take over a terminal body of another uuid. +TEST(CASMount, ClaimPathsReclaimAForeignTerminalBodyOfTheirOwnOwner) +{ + { + SCOPED_TRACE("start"); + auto b = std::make_shared(); + Layout l("p"); + { auto first = Pool::open(b, rootConfig()); } + rewriteSlotUuid(*b, l, "r", UInt128(2)); + auto store = Pool::open(b, rootConfig()); + const MountLease slot = slotBody(*b, l, "r"); + EXPECT_EQ(slot.server_uuid, UInt128(1)); + EXPECT_EQ(slot.writer_epoch, store->writerEpoch()); + } + { + SCOPED_TRACE("self-remount"); + auto b = std::make_shared(); + Layout l("p"); + auto store = Pool::open(b, rootConfig()); + DB::Cas::tests::fenceOutMountForTest(*b, l.mountKey("r")); + DB::Cas::tests::expectThrowsCode(DB::ErrorCodes::ABORTED, [&] { store->renewWatermarkOnce(); }); + rewriteSlotUuid(*b, l, "r", UInt128(2)); + EXPECT_TRUE(store->tryRemountOnce()); + const MountLease slot = slotBody(*b, l, "r"); + EXPECT_EQ(slot.server_uuid, UInt128(1)); + EXPECT_FALSE(slot.gc_fenced); + } +} + +/// A terminal row reads its owner: a tombstone makes it `retired`, and the row carries whether the retirement is +/// verified. A live row never reads its owner (its owner here is undecodable). +TEST(CASListMounts, TerminalSlotOfATombstonedOwnerIsRetired) +{ + const auto state_of = [](Ops & ops, const Layout & l, const String & srid) -> String + { + for (const MountInfo & m : listMounts(ops.op, l, /*now_ms=*/0, /*skew_margin_ms=*/0)) + if (m.srid == srid) + return m.state + (m.retirement_verified ? "+verified" : ""); + return "absent"; + }; + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + { auto first = Pool::open(b, rootConfig()); } + EXPECT_EQ(state_of(ops, l, "r"), "terminated"); + DB::Cas::tests::tombstoneOwnerForTest(*b, l.ownerKey("r")); + EXPECT_EQ(state_of(ops, l, "r"), "retired") << "an unverified tombstone"; + DB::Cas::tests::tombstoneOwnerForTest(*b, l.ownerKey("r"), /*verified=*/true); + EXPECT_EQ(state_of(ops, l, "r"), "retired+verified"); + DB::Cas::tests::fenceOutMountForTest(*b, l.mountKey("r")); + EXPECT_EQ(state_of(ops, l, "r"), "retired+verified") << "a fenced slot of a retired owner"; + rewriteOwnerAt(*b, l, "r", UInt128(2)); + EXPECT_EQ(state_of(ops, l, "r"), "fenced") << "a rejoined owner"; + + mustCommit(ops.op.create(l.ownerKey("x"), "garbage-not-a-proto", Retry::standard()), "undecodable owner of x"); + mustCommit(ops.op.create(l.mountKey("x"), encodeMountLease(liveBody(UInt128(1), 4)), Retry::standard()), "live slot of x"); + EXPECT_EQ(state_of(ops, l, "x"), "live") << "a non-terminal row keeps its lease state and never reads its owner"; +} + +TEST(CASListMounts, TerminalSlotWithAnUndecodableOwnerIsCorrupt) +{ + auto b = std::make_shared(); + Layout l("p"); + Ops ops(b); + { auto first = Pool::open(b, rootConfig()); } + DB::Cas::tests::OperationForTest op(*b); + const auto current = (*op).read(l.ownerKey("r"), Retry::once()); + ASSERT_TRUE(current.has_value()); + ASSERT_TRUE(std::holds_alternative((*op).replace(l.ownerKey("r"), "garbage-not-a-proto", current->etag, Retry::once()))); + mustCommit(ops.op.create(l.mountKey("other"), encodeMountLease(liveBody(UInt128(1), 4)), Retry::standard()), "live slot of other"); + std::map by_srid; + for (const MountInfo & m : listMounts(ops.op, l, /*now_ms=*/0, /*skew_margin_ms=*/0)) + by_srid[m.srid] = m.state; + EXPECT_EQ(by_srid.size(), 2u); + EXPECT_EQ(by_srid["r"], "corrupt"); + EXPECT_EQ(by_srid["other"], "live"); +} diff --git a/src/Disks/tests/gtest_cas_mount_claim_conflicts.cpp b/src/Disks/tests/gtest_cas_mount_claim_conflicts.cpp index 6eb8f9c9b7b85..d5aa0235b6be6 100644 --- a/src/Disks/tests/gtest_cas_mount_claim_conflicts.cpp +++ b/src/Disks/tests/gtest_cas_mount_claim_conflicts.cpp @@ -116,8 +116,14 @@ TEST(CASMountClaimConflicts, SlotChangedInsideAdoptionWindow) /// Rewrite the slot under a NEW incarnation after our read, so our adoption write is refused. backend->before_put_overwrite = [&] { - CasOperation racer = requests.admit(); - claimMount(racer, layout, "r", DB::UInt128(1), 7, now + 1, /*ttl_ms=*/100); + OperationForTest racer(*backend); + const String key = layout.mountKey("r"); + const auto got = (*racer).read(key, Retry::standard()); + ASSERT_TRUE(got.has_value()); + MountLease rewritten = decodeMountLease(got->bytes); + ++rewritten.seq; + ASSERT_TRUE(std::holds_alternative( + (*racer).replace(key, encodeMountLease(rewritten), got->etag, Retry::standard()))); }; auto renewer = makeRenewer(requests, now); expectThrowsCodeWithMessage( @@ -216,9 +222,8 @@ TEST(CASMountClaimConflicts, ALostCreateReportsTheOccupantNotTheProposer) << "the operator message must name the foreign holder"; } -/// The same for the refresh branch: a body that changed under our own adoption is the one the message -/// must name. The reclaim branch reaches the identical helper, so it is not repeated here. -TEST(CASMountClaimConflicts, ALostRefreshReportsTheObservedBodyNotTheProposer) +/// The same for the reclaim branch: a body that changed under our own reclaim is the one the message must name. +TEST(CASMountClaimConflicts, ALostReclaimReportsTheObservedBodyNotTheProposer) { auto backend = std::make_shared(); Layout layout("p"); @@ -227,7 +232,8 @@ TEST(CASMountClaimConflicts, ALostRefreshReportsTheObservedBodyNotTheProposer) CasOperation seed = requests.admit(); ASSERT_EQ(claimMount(seed, layout, "r", DB::UInt128(1), 7, now, /*ttl_ms=*/100).kind, MountClaimResult::Claimed); - /// A distinguishable body lands under our own refresh, so a result carrying the proposal cannot + markMountGcFenced(seed, layout, "r"); + /// A distinguishable body lands under our own reclaim, so a result carrying the proposal cannot /// pass by accident: our proposal would carry `seq` 2 and this process's pid. backend->before_put_overwrite = [&] { @@ -242,7 +248,7 @@ TEST(CASMountClaimConflicts, ALostRefreshReportsTheObservedBodyNotTheProposer) (*racer).replace(key, encodeMountLease(raced), got->etag, Retry::standard()))); }; CasOperation op = requests.admit(); - const MountClaimResult claim = claimMount(op, layout, "r", DB::UInt128(1), 7, now, /*ttl_ms=*/100); + const MountClaimResult claim = claimMount(op, layout, "r", DB::UInt128(1), 8, now, /*ttl_ms=*/100); EXPECT_EQ(claim.kind, MountClaimResult::LiveDoubleStart); ASSERT_TRUE(claim.body.has_value()); @@ -256,7 +262,7 @@ TEST(CASMountClaimConflicts, ALostRefreshReportsTheObservedBodyNotTheProposer) /// process in the "Existing mount" line of an operator message -- the same defect as naming it after a /// conflict that did observe someone. `ProvenAbsent` is the reachable half; `NotObserved` (the resolve /// read itself failed) leaves through the same branch. -TEST(CASMountClaimConflicts, ARacedRefreshThatObservedNothingNamesNoHolder) +TEST(CASMountClaimConflicts, ARacedReclaimThatObservedNothingNamesNoHolder) { auto backend = std::make_shared(); Layout layout("p"); @@ -265,7 +271,8 @@ TEST(CASMountClaimConflicts, ARacedRefreshThatObservedNothingNamesNoHolder) CasOperation seed = requests.admit(); ASSERT_EQ(claimMount(seed, layout, "r", DB::UInt128(1), 7, now, /*ttl_ms=*/100).kind, MountClaimResult::Claimed); - /// The slot is removed under our own refresh, so the refused precondition resolves to a proven + markMountGcFenced(seed, layout, "r"); + /// The slot is removed under our own reclaim, so the refused precondition resolves to a proven /// absence rather than to an occupant. backend->before_put_overwrite = [&] { @@ -276,7 +283,7 @@ TEST(CASMountClaimConflicts, ARacedRefreshThatObservedNothingNamesNoHolder) ASSERT_EQ((*racer).remove(key, got->etag, Retry::standard()), Removal::Removed); }; CasOperation op = requests.admit(); - const MountClaimResult claim = claimMount(op, layout, "r", DB::UInt128(1), 7, now, /*ttl_ms=*/100); + const MountClaimResult claim = claimMount(op, layout, "r", DB::UInt128(1), 8, now, /*ttl_ms=*/100); EXPECT_EQ(claim.kind, MountClaimResult::LiveDoubleStart); EXPECT_FALSE(claim.etag.has_value()); diff --git a/src/Disks/tests/gtest_cas_mount_renewal_progress.cpp b/src/Disks/tests/gtest_cas_mount_renewal_progress.cpp new file mode 100644 index 0000000000000..97d298d02b5ee --- /dev/null +++ b/src/Disks/tests/gtest_cas_mount_renewal_progress.cpp @@ -0,0 +1,107 @@ +#include + +#include +#include +#include +#include + +#include + +#include "config.h" + +#if USE_AWS_S3 +#include +#endif + +namespace ProfileEvents +{ +extern const Event CASMountRenewalAttempts; +} + +using namespace DB::Cas; + +namespace +{ + +uint64_t attemptsCounter() +{ + return ProfileEvents::global_counters[ProfileEvents::CASMountRenewalAttempts]; +} + +/// Fixed boot clock and no-op sleep: every armed fault is finite, so retries end without time passing. +struct Fixture +{ + std::shared_ptr backend = std::make_shared(); + PoolPtr store; + String mount_key; + + explicit Fixture(const String & prefix) + { + backend->setAttemptTimeoutMs(10); + store = Pool::open(backend, PoolConfig{ + .pool_prefix = prefix, + .server_root_id = "test", + .mount_lease_ttl_ms = std::chrono::milliseconds(1000), + .cas_request_budget = CasRequestBudget{.attempt_timeout_ms = 10, .lease_safety_margin_ms = 20, .connect_timeout_cap_ms = std::nullopt}, + .boot_ms_fn = [] { return uint64_t{100}; }, + .retry_sleep_fn = [](uint64_t) {}, + }); + mount_key = store->layout().mountKey("test"); + } +}; + +} + +TEST(CASMountRenewalProgress, AttemptsAndLastErrorAreVisibleWhileTheRenewalRuns) +{ + Fixture f("renewal-progress-in-flight"); + const auto timeout = std::make_exception_ptr(Poco::TimeoutException("injected renewal timeout")); + f.backend->failNextWriteWith(f.mount_key, timeout); + f.backend->failNextWriteWith(f.mount_key, timeout); + const uint64_t attempts_before = attemptsCounter(); + /// Read at the third attempt, the first that reaches the store. Heap state and a raw pointer: the + /// hook must not keep the pool alive, and it is removed before the pool's farewell write. + auto in_flight = std::make_shared>>(); + Pool * pool = f.store.get(); + f.backend->onBeforeWrite(f.mount_key, [in_flight, pool] + { + if (!*in_flight) + *in_flight = std::make_pair(pool->renewalProgress(), attemptsCounter()); + }); + + f.store->renewWatermarkOnce(); + f.backend->onBeforeWrite(f.mount_key, {}); + + ASSERT_TRUE(in_flight->has_value()); + const auto & [seen, attempts_counted] = **in_flight; + EXPECT_TRUE(seen.in_progress); + EXPECT_EQ(seen.attempts_sent, 3u); + EXPECT_EQ(seen.last_error, "PUT " + f.mount_key + ": transport Timeout injected renewal timeout"); + EXPECT_EQ(attempts_counted - attempts_before, 3u) << "attempts are counted as they are sent"; + + const RenewalProgress ended = f.store->renewalProgress(); + EXPECT_FALSE(ended.in_progress); + EXPECT_EQ(ended.attempts_sent, 3u); + EXPECT_EQ(ended.last_error, seen.last_error) << "an ended renewal keeps its progress"; + + f.store->renewWatermarkOnce(); + const RenewalProgress next = f.store->renewalProgress(); + EXPECT_EQ(next.attempts_sent, 1u); + EXPECT_FALSE(next.last_error.has_value()) << "the next renewal starts clean"; +} + +#if USE_AWS_S3 +TEST(CASMountRenewalProgress, AStoreRefusalIsClassedRefused) +{ + Fixture f("renewal-progress-refused"); + f.backend->failNextWriteWith(f.mount_key, std::make_exception_ptr( + DB::S3Exception("the store refused the renewal", Aws::S3::S3Errors::ACCESS_DENIED, "AccessDenied"))); + + EXPECT_THROW(f.store->renewWatermarkOnce(), DB::Exception); + + const RenewalProgress ended = f.store->renewalProgress(); + EXPECT_FALSE(ended.in_progress) << "a renewal that threw has still ended"; + EXPECT_EQ(ended.attempts_sent, 1u); + EXPECT_EQ(ended.last_error, "PUT " + f.mount_key + ": refused AccessDenied the store refused the renewal"); +} +#endif diff --git a/src/Disks/tests/gtest_cas_orphan_manifest_sweep.cpp b/src/Disks/tests/gtest_cas_orphan_manifest_sweep.cpp index 41bc8258a47d8..65b1e9388cec4 100644 --- a/src/Disks/tests/gtest_cas_orphan_manifest_sweep.cpp +++ b/src/Disks/tests/gtest_cas_orphan_manifest_sweep.cpp @@ -12,10 +12,12 @@ #include "cas_sweep_test_support.h" #include "cas_test_helpers.h" #include +#include #include #include #include #include +#include using namespace DB::Cas; using namespace DB::Cas::tests; @@ -179,6 +181,72 @@ TEST(CASOrphanManifestSweep, EligibleAndUnownedIsDeleted) EXPECT_FALSE(headExists(*backend, store->layout().manifestKey(ManifestId{ns, r}))); } +/// The decommission drain runs this on its admin pool: once that pool lost its mount, nothing is deleted. +TEST(CASOrphanManifestSweep, LostMountDeletesNothing) +{ + auto backend = std::make_shared(); + auto store = openPoolForTest(backend); + const RootNamespace ns{"00/aa@cas@"}; + registerNamespaceRaw(*backend, store->layout(), ns); + const ManifestRef r = ref(5, 0xAB); + writeManifestRaw(*backend, store->layout(), ns, r, {blobEntryFor("a", DB::UInt128(1))}); + setWatermarkMinActive(*backend, store->layout(), kServerRoot, kWriterEpoch, /*min_active_build_sequence*/6); + seedConsumedSealCursor(*backend, store->layout(), ns); + seedEmptyRecoveryAuthority(*backend, store->layout(), ns); + store->tripMountLost(); + + std::vector warnings; + EXPECT_THROW(sweepNamespace(*store, ns, BuildPrefix{.writer_epoch = kWriterEpoch, .build_sequence = 5}, &warnings), + DB::Exception); + EXPECT_TRUE(headExists(*backend, store->layout().manifestKey(ManifestId{ns, r}))); +} + +namespace +{ +/// Runs `on_first_manifest_remove` once, inside the first delete of a manifest, after it applied. +class HookOnFirstManifestRemoveBackend : public InMemoryBackend +{ +public: + std::function on_first_manifest_remove; + + RawRemoval remove(const String & key, const String & expected_value, TransportAccess & access) override + { + const RawRemoval removed = InMemoryBackend::remove(key, expected_value, access); + if (on_first_manifest_remove && key.starts_with("p/cas/manifests/")) + std::exchange(on_first_manifest_remove, {})(); + return removed; + } +}; +} + +/// The mount is lost after the first delete: the walk stops at the first refused key instead of adding one +/// warning per remaining key. +TEST(CASOrphanManifestSweep, LostMountStopsTheWalkAtTheFirstRefusal) +{ + auto backend = std::make_shared(); + auto store = openPoolForTest(backend); + const RootNamespace ns{"00/aa@cas@"}; + registerNamespaceRaw(*backend, store->layout(), ns); + std::vector keys; + for (const uint64_t ordinal : {0xABu, 0xACu, 0xADu, 0xAEu}) + { + const ManifestRef r = ref(5, ordinal); + writeManifestRaw(*backend, store->layout(), ns, r, {blobEntryFor("a", DB::UInt128(1))}); + keys.push_back(store->layout().manifestKey(ManifestId{ns, r})); + } + setWatermarkMinActive(*backend, store->layout(), kServerRoot, kWriterEpoch, /*min_active_build_sequence*/6); + seedConsumedSealCursor(*backend, store->layout(), ns); + seedEmptyRecoveryAuthority(*backend, store->layout(), ns); + backend->on_first_manifest_remove = [&] { store->tripMountLost(); }; + + std::vector warnings; + sweepNamespace(*store, ns, BuildPrefix{.writer_epoch = kWriterEpoch, .build_sequence = 5}, &warnings); + + EXPECT_EQ(warnings.size(), 1u) << fmt::format("{}", fmt::join(warnings, "; ")); + const auto kept = std::count_if(keys.begin(), keys.end(), [&](const String & key) { return headExists(*backend, key); }); + EXPECT_GE(kept, 2); +} + /// The orphan sweep must not turn a forged same-id snapshot at an OLDER `EpochSeal` into an empty owner /// view. The base differs from `last_epoch_seal`, so metadata equality cannot catch it; the candidate /// remains retained until the checkpoint is repaired. diff --git a/src/Disks/tests/gtest_cas_part_write.cpp b/src/Disks/tests/gtest_cas_part_write.cpp index a60f5b45b11ad..78a1fb7a54d0f 100644 --- a/src/Disks/tests/gtest_cas_part_write.cpp +++ b/src/Disks/tests/gtest_cas_part_write.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,7 @@ namespace ProfileEvents extern const Event CASMetaPut; extern const Event CASMetaCompareSwap; extern const Event CASMetaCreateClean; -extern const Event CASMetaAdoptBackfill; +extern const Event CASMetaAbsentRepublish; extern const Event CASMetaResurrectClean; extern const Event CASBlobAdoptTrusted; } @@ -512,44 +513,32 @@ TEST(CASPartWriteTxnMetaCounters, CreateCleanAndChokePointCountOnFreshBody) EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASMetaCreateClean] - reason_before, 1); } -/// §0 introspection: an adopt of a pre-existing body that has NO meta at all (a pre-protocol blob, or a -/// lost race with a concurrent fresh-uploader's own meta write) backfills a Clean meta through the -/// `putMetaIfAbsent` choke point (`CASMetaPut`), tagged with its reason (`CASMetaAdoptBackfill`). No -/// existing test elsewhere in the suite drives this branch: every other pre-seeded raw body in this file -/// pairs `writeRawBlobBody` with `writeMetaClean`, which skips the `!lm` backfill branch entirely. -TEST(CASPartWriteTxnMetaCounters, AdoptBackfillCountsChokePointAndReason) +/// A present body without a marker is republished, and the marker is created through the same choke +/// point (`CASMetaPut`) and reason (`CASMetaCreateClean`) as for an absent body. +TEST(CASPartWriteTxnMetaCounters, AbsentMarkerRepublishCountsChokePointAndReason) { const auto put_before = ProfileEvents::global_counters[ProfileEvents::CASMetaPut]; - const auto reason_before = ProfileEvents::global_counters[ProfileEvents::CASMetaAdoptBackfill]; + const auto create_before = ProfileEvents::global_counters[ProfileEvents::CASMetaCreateClean]; + const auto republish_before = ProfileEvents::global_counters[ProfileEvents::CASMetaAbsentRepublish]; auto b = std::make_shared(); auto s = openPool(b); - const String payload = "adopt-backfill-payload-counters"; + const String payload = "absent-marker-republish-counters"; const UInt128 hash = u128Of(payload); const BlobRef id = idOf(payload); - - /// Pre-seed a present body big enough that `ensureBlobPresent`'s logical-size guard does not underflow — - /// deliberately WITHOUT any meta (unlike PutBlobAdoptsWhenMetaCleanNoRetireView), so the adopt reaches - /// the `!lm` backfill branch. - const uint64_t header_len = s->poolMeta().blob_header_len; - String raw_body(header_len, '\0'); + String raw_body(s->poolMeta().blob_header_len, '\0'); raw_body += payload; writeRawBlobBody(*b, s->layout(), hash, raw_body); - /// Adopt must happen under a durable precommit edge (EDGE-BEFORE-OBSERVE). - const RootNamespace ns{"srv/tbl"}; - auto build = startBuildFor(s, ns, "ref_adopt_backfill"); - const ManifestId manifest_id = build->stageManifest({blobManifestEntry("data.bin", payload)}); - build->precommitAdd(ns, "ref_adopt_backfill", manifest_id); - auto ref = build->putBlob(id, BlobSource::fromString(payload)); - EXPECT_EQ(ref.ref, id); + auto build = precommittedBuildForPayload(s, RootNamespace{"srv1/absent-marker-counters"}, "part", payload); + EXPECT_EQ(build->putBlob(id, BlobSource::fromString(payload)).ref, id); EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASMetaPut] - put_before, 1); - EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASMetaAdoptBackfill] - reason_before, 1); - + EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASMetaCreateClean] - create_before, 1); + EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASMetaAbsentRepublish] - republish_before, 1); const auto lm = loadMetaForTest(*b, s->layout(), hash); - ASSERT_TRUE(lm.has_value()) << "the adopt-backfill must leave a Clean meta for future point-readers"; + ASSERT_TRUE(lm.has_value()); EXPECT_EQ(lm->meta.state, MetaState::Clean); } @@ -2712,11 +2701,10 @@ TEST(CASPartWrite, AmbiguousTimeoutsThenCommitRestreamsFromSource) EXPECT_TRUE((*op).head(s->layout().blobKey(idOf(payload)), Retry::once()).has_value()) << "the blob body must be durable"; } -/// Ambiguous-but-landed: the FIRST attempt's response is lost AFTER the write actually landed -/// server-side. The occupancy resolve observes the key present and the existing 412 machinery takes -/// over — the occupant is ADOPTED (content-addressed identity: any occupant of this key IS the -/// content), with NO reissue and NO second body upload. -TEST(CASPartWrite, AmbiguousLandedWriteAdoptsOccupantWithoutReupload) +/// Ambiguous-but-landed: the first publication's response is lost after the body landed, so the +/// marker was never created. A present body without a marker is never adopted, so the landed attempt +/// costs one more publication, after which the marker is `Clean`. +TEST(CASPartWrite, AmbiguousLandedWriteRepublishesOnceThenClean) { auto b = std::make_shared(); /// Heap-owned, not a plain local: `~Pool` emits terminate events into the sink, and a background @@ -2742,24 +2730,19 @@ TEST(CASPartWrite, AmbiguousLandedWriteAdoptsOccupantWithoutReupload) const PutBlobResult res = build->putBlob(idOf(payload), countingSource(payload, payload_streams)); EXPECT_EQ(res.size, payload.size()); - EXPECT_EQ(b->publish_stream_attempts, 1) << "a landed ambiguous attempt must be observed, never reissued"; - EXPECT_EQ(b->blob_head_attempts, 2) << "the ambiguity is resolved by restarting from HEAD"; - EXPECT_EQ(payload_streams, 1); + EXPECT_EQ(b->publish_stream_attempts, 2) << "the landed attempt and exactly one republication"; + EXPECT_EQ(b->blob_head_attempts, 2); + EXPECT_EQ(payload_streams, 2); - const String key = s->layout().blobKey(idOf(payload)); + const auto marker = loadMetaForTest(*b, s->layout(), u128Of(payload)); + ASSERT_TRUE(marker.has_value()); + EXPECT_EQ(marker->meta.state, MetaState::Clean); const std::vector observed_events = events->snapshot(); - const auto adopt = std::find_if(observed_events.begin(), observed_events.end(), - [](const CasEvent & e) { return e.type == CasEventType::BlobReuseAdopt; }); - ASSERT_NE(adopt, observed_events.end()) << "the landed occupant must be ADOPTED (the standard dedup leg)"; - CasRequests probe(b, Fence::open()); - CasOperation probe_op = probe.admit(); - const auto landed = probe_op.head(key, Retry::standard()); - ASSERT_TRUE(landed.has_value()); - EXPECT_EQ(adopt->token, landed->etag.render()) - << "the adopted token must be the landed incarnation, rendered"; EXPECT_EQ(std::count_if(observed_events.begin(), observed_events.end(), - [](const CasEvent & e) { return e.type == CasEventType::BlobPut; }), 0) - << "no fresh-upload event: the body was never re-uploaded"; + [](const CasEvent & e) { return e.type == CasEventType::BlobReuseAdopt; }), 0) + << "a body without a marker was adopted"; + EXPECT_EQ(std::count_if(observed_events.begin(), observed_events.end(), + [](const CasEvent & e) { return e.type == CasEventType::BlobPut; }), 1); } /// Every physical publication mints its own envelope, and the request engine never reissues one. Two @@ -2925,16 +2908,12 @@ TEST(CASPartWrite, AmbiguousNonLandingPublicationStopsAtOuterBound) EXPECT_EQ(payload_streams, 8); } -/// A server-side copy publication is ambiguous-but-landed: its response is lost after the -/// destination was created. The occupancy resolve observes the destination present and the occupant -/// is adopted — Committed-in-effect WITHOUT a re-copy. -TEST(CASPartWrite, AmbiguousCopyLandedAdoptsDestinationWithoutRecopy) +/// A server-side copy publication is ambiguous-but-landed: its response is lost after the destination +/// was created, so the marker was never created. The verbatim copy is spent, so the one republication +/// streams the payload under a fresh envelope, after which the marker is `Clean`. +TEST(CASPartWrite, AmbiguousCopyLandedRepublishesOnceThenClean) { auto b = std::make_shared(); - /// Heap-owned, not a plain local: `~Pool` emits terminate events into the sink, and a background - /// publish can hold an extra `shared_from_this()` past this frame's return regardless of - /// declaration order relative to the Pool, so a by-reference capture of a local would dangle. - auto events = std::make_shared(); auto s = openBlobFaultPool(b); const RootNamespace ns{"srv/tbl"}; const String payload = "staged-payload-A"; @@ -2946,11 +2925,6 @@ TEST(CASPartWrite, AmbiguousCopyLandedAdoptsDestinationWithoutRecopy) ASSERT_TRUE(std::holds_alternative((*seed_op).create(staging_key, staging_bytes, Retry::once()))); } - s->setEventSink([events](const CasEvent & e) - { - events->push(e); - }); - auto build = startBuildFor(s, ns, "part_copy_landed"); const ManifestId id = build->stageManifest({blobManifestEntry("a.bin", payload)}); build->precommitAdd(ns, "part_copy_landed", id); @@ -2958,23 +2932,27 @@ TEST(CASPartWrite, AmbiguousCopyLandedAdoptsDestinationWithoutRecopy) BlobSource source; source.size = payload.size(); source.server_side_copy_from = staging_key; + source.open = [payload]() -> std::unique_ptr + { + return std::make_unique(payload); + }; b->fault_count = 1; b->land_despite_fault = true; const PutBlobResult res = build->putBlob(idOf(payload), std::move(source)); EXPECT_EQ(res.size, payload.size()); - EXPECT_EQ(b->publish_copy_attempts, 1) << "a landed ambiguous copy must be resolved, never re-copied"; - EXPECT_EQ(b->publish_stream_attempts, 0); + EXPECT_EQ(b->publish_copy_attempts, 1) << "the verbatim copy is a once-only privilege"; + EXPECT_EQ(b->publish_stream_attempts, 1) << "one republication after the landed copy"; EXPECT_EQ(b->blob_head_attempts, 2); const String key = s->layout().blobKey(idOf(payload)); OperationForTest op(*b); const auto got = (*op).read(key, Retry::once()); ASSERT_TRUE(got.has_value()); - EXPECT_EQ(got->bytes, staging_bytes) << "the destination is the staging object's verbatim copy"; - const std::vector observed_events = events->snapshot(); - EXPECT_NE(std::find_if(observed_events.begin(), observed_events.end(), - [](const CasEvent & e) { return e.type == CasEventType::BlobReuseAdopt; }), - observed_events.end()) << "the landed destination must be ADOPTED"; + EXPECT_NE(got->bytes, staging_bytes) << "the landed copy was adopted"; + EXPECT_EQ(got->bytes.substr(s->poolMeta().blob_header_len), payload); + const auto marker = loadMetaForTest(*b, s->layout(), u128Of(payload)); + ASSERT_TRUE(marker.has_value()); + EXPECT_EQ(marker->meta.state, MetaState::Clean); } /// A server-side copy publication is ambiguous-and-absent: the first copy attempt times out with @@ -3018,3 +2996,117 @@ TEST(CASPartWrite, AmbiguousCopyAbsentReattemptsAndCommits) EXPECT_NE(got->bytes, staging_bytes); EXPECT_EQ(got->bytes.substr(s->poolMeta().blob_header_len), payload); } + +namespace +{ + +/// Runs a side effect once, on the first read of the watched key, before that read is served. +class BeforeMetaReadBackend final : public InMemoryBackend +{ +public: + String watched_key; + std::function trigger; + + std::optional read(const String & key, TransportAccess & access) override + { + if (key == watched_key && trigger) + std::exchange(trigger, {})(); + return InMemoryBackend::read(key, access); + } +}; + +/// A present body and no marker, as a writer that crashed between its publication and its marker +/// create leaves them. +void seedBodyWithoutMarker(Backend & backend, const PoolPtr & store, const String & payload) +{ + String raw_body(store->poolMeta().blob_header_len, '\0'); + raw_body += payload; + writeRawBlobBody(backend, store->layout(), u128Of(payload), raw_body); +} + +BlobUploadResult uploadOnce(const PartWriteTxnPtr & build, const BlobRef & ref, const String & payload) +{ + return build->uploadBlobDetached(BlobUploadRequest{ref, BlobSource::fromString(payload), payload.size()}); +} + +} + +/// GC deletes the body and then its marker between the writer's `HEAD` and its marker read. The writer +/// saw a present body and an absent marker; adopting would name a deleted body. +TEST(CASPartWriteTxnMarkerRace, AbsentMarkerAfterHeadRepublishes) +{ + auto b = std::make_shared(); + auto s = openPool(b); + const String payload = "absent-marker-after-head"; + const BlobRef ref = idOf(payload); + const String blob_key = s->layout().blobKey(ref); + seedBodyWithoutMarker(*b, s, payload); + writeMetaClean(*b, s->layout(), u128Of(payload), payload.size()); + condemnMeta(*b, s->layout(), u128Of(payload), /*condemn_round*/ 1); + const Etag condemned_token = [&] + { + OperationForTest op(*b); + return (*op).head(blob_key, Retry::once()).value().etag; + }(); + + auto build = precommittedBuildForPayload(s, RootNamespace{"srv1/marker-race"}, "part", payload); + bool fired = false; + b->watched_key = s->layout().blobMetaKey(ref); + b->trigger = [&] + { + fired = true; + OperationForTest op(*b); + ASSERT_EQ((*op).remove(blob_key, condemned_token, Retry::once()), Removal::Removed); + const auto marker = loadMeta(*op, s->layout(), ref); + ASSERT_TRUE(marker.has_value()); + ASSERT_EQ(deleteMetaExact(*op, s->layout(), ref, marker->etag), Removal::Removed); + }; + + const BlobUploadResult result = uploadOnce(build, ref, payload); + ASSERT_TRUE(fired) << "the writer never read the marker after its `HEAD`"; + + EXPECT_EQ(result.diagnostics.action, BlobMaterializationAction::Published) << "adopted a deleted body"; + EXPECT_EQ(result.diagnostics.reason, std::optional(BlobPublicationReason::Absent)); + OperationForTest op(*b); + const auto head = (*op).head(blob_key, Retry::once()); + ASSERT_TRUE(head.has_value()) << "the writer's dependency names an absent body"; + EXPECT_NE(head->etag, condemned_token); + const auto marker = loadMetaForTest(*b, s->layout(), u128Of(payload)); + ASSERT_TRUE(marker.has_value()); + EXPECT_EQ(marker->meta.state, MetaState::Clean); + + /// GC's exact-token delete of the condemned incarnation, replayed late, misses the fresh body. + EXPECT_EQ((*op).remove(blob_key, condemned_token, Retry::once()), Removal::Mismatch); + EXPECT_TRUE((*op).head(blob_key, Retry::once()).has_value()); +} + +/// A crash leftover costs one upload: the first writer republishes and creates the marker, the next one +/// adopts on that `Clean` marker. +TEST(CASPartWriteTxnMarkerRace, PresentBodyWithoutMarkerRepublishesOnce) +{ + auto b = std::make_shared(); + auto s = openPool(b); + const String payload = "present-body-without-marker"; + const BlobRef ref = idOf(payload); + const String blob_key = s->layout().blobKey(ref); + seedBodyWithoutMarker(*b, s, payload); + OperationForTest op(*b); + const Etag leftover = (*op).head(blob_key, Retry::once())->etag; + + const auto creates_before = ProfileEvents::global_counters[ProfileEvents::CASMetaCreateClean]; + auto first = precommittedBuildForPayload(s, RootNamespace{"srv1/leftover-a"}, "part", payload); + const BlobUploadResult first_result = uploadOnce(first, ref, payload); + EXPECT_EQ(first_result.diagnostics.action, BlobMaterializationAction::Published); + EXPECT_EQ(first_result.diagnostics.reason, std::optional(BlobPublicationReason::Absent)); + EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASMetaCreateClean] - creates_before, 1); + const auto republished = (*op).head(blob_key, Retry::once()); + ASSERT_TRUE(republished.has_value()); + EXPECT_NE(republished->etag, leftover); + const auto marker = loadMetaForTest(*b, s->layout(), u128Of(payload)); + ASSERT_TRUE(marker.has_value()); + EXPECT_EQ(marker->meta.state, MetaState::Clean); + + auto second = precommittedBuildForPayload(s, RootNamespace{"srv1/leftover-b"}, "part", payload); + EXPECT_EQ(uploadOnce(second, ref, payload).diagnostics.action, BlobMaterializationAction::Observed); + EXPECT_EQ((*op).head(blob_key, Retry::once())->etag, republished->etag); +} diff --git a/src/Disks/tests/gtest_cas_pool.cpp b/src/Disks/tests/gtest_cas_pool.cpp index 63e0d917f9dba..b60f049e58dda 100644 --- a/src/Disks/tests/gtest_cas_pool.cpp +++ b/src/Disks/tests/gtest_cas_pool.cpp @@ -16,10 +16,13 @@ #include #include #include +#include #include #include +#include "config.h" #include #include +#include #include #include #include @@ -52,6 +55,7 @@ extern const Event CASMountExclusivityViolation; extern const Event CASMountLeaseLost; extern const Event CASMountReleaseSkippedForeignOccupant; extern const Event CASRemountAttempts; +extern const Event CASIdentityLost; extern const Event CASRemountSucceeded; extern const Event CASRemountFailed; } @@ -1651,6 +1655,19 @@ TEST(CASPoolRemount, FenceOutThenSelfRemountRestoresWrites) EXPECT_NO_THROW(store->renewWatermarkOnce()); } +TEST(CASPool, RestartKeepsTheOwnerLife) +{ + auto backend = std::make_shared(); + String owner_key; + { auto store = DB::Cas::tests::openPoolForTest(backend); owner_key = store->layout().ownerKey(store->poolConfig().server_root_id); } + DB::Cas::tests::OperationForTest op(*backend); + const auto before = (*op).read(owner_key, DB::Cas::Retry::once()); + ASSERT_TRUE(before.has_value()); + EXPECT_EQ(DB::Cas::decodeOwner(before->bytes).owner_life, 1u); + auto again = DB::Cas::tests::openPoolForTest(backend); + EXPECT_EQ((*op).read(owner_key, DB::Cas::Retry::once())->etag, before->etag) << "a plain restart writes no owner"; +} + TEST(CASPoolRemount, OldEpochBuildFailsClosedAfterRemount) { auto backend = std::make_shared(); @@ -1733,6 +1750,38 @@ TEST(CASPoolRemount, ForeignOwnerIsNeverTakenOver) << "the slot must be left byte-for-byte as the foreign owner wrote it"; } +/// The background remount loop over a retired owner: one attempt ends in `IdentityLost`, counts one +/// `CASIdentityLost`, and the worker exits instead of backing off and retrying. +TEST(CASPoolRemount, RemountLoopEndsOverARetiredOwner) +{ + auto backend = std::make_shared(); + auto first_attempt = std::make_shared>(); + auto attempts = std::make_shared>(0); + auto store = Pool::open(backend, PoolConfig{ + .pool_prefix = "remount-loop-retired-owner", + .server_root_id = "test", + .background_watermark = true, + .event_sink = [first_attempt, attempts](const CasEvent & event) + { + if (event.type == CasEventType::MountRemount && attempts->fetch_add(1) == 0) + first_attempt->set_value(); + }, + }); + const String owner_key = store->layout().ownerKey("test"); + const String mount_key = store->layout().mountKey("test"); + const uint64_t identity_lost_before = ProfileEvents::global_counters[ProfileEvents::CASIdentityLost]; + + DB::Cas::tests::tombstoneOwnerForTest(*backend, owner_key); + fenceOutMount(*backend, mount_key); + ASSERT_TRUE(store->scheduleRemountForTest()); + ASSERT_EQ(first_attempt->get_future().wait_for(std::chrono::seconds(60)), std::future_status::ready); + EXPECT_EQ(store->lifecycle(), PoolLifecycle::IdentityLost); + + store.reset(); /// joins both workers + EXPECT_EQ(attempts->load(), 1u) << "the loop did not retry"; + EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASIdentityLost], identity_lost_before + 1); +} + TEST(CASPoolRemount, ShutdownGuardRefusesToArmRemount) { auto backend = std::make_shared(); @@ -2026,6 +2075,16 @@ class RuntimeRenewBackend final : public DB::Cas::tests::CountingBackend /// rather than reissued has to move the injected clock here -- from inside the attempt, which is the /// only point between admission and the resolve read a test can reach. std::function before_throw; + /// While it answers true, every guarded write fails with a transport timeout before `fault` is + /// consulted. Set it before the workers start. + std::function outage; + /// Runs inside every write `outage` fails, before it throws. + std::function on_outage_write; + std::atomic outage_writes{0}; + /// Runs first in every guarded write. + std::function on_guarded_write; + /// While positive, every guarded write is refused with an access denial. + std::atomic deny_writes{0}; /// The fault sits on the WRITE PRIMITIVE, and only on a CONDITIONAL one: a lease renewal is a /// replace, so a create on the same key must not consume the one-shot fault. @@ -2034,6 +2093,24 @@ class RuntimeRenewBackend final : public DB::Cas::tests::CountingBackend { if (!expected_value) return DB::Cas::tests::CountingBackend::write(key, bytes, expected_value, access); + if (on_guarded_write) + on_guarded_write(); + if (deny_writes.load() > 0) + { + deny_writes.fetch_sub(1); +#if USE_AWS_S3 + throw DB::S3Exception("injected mount PUT access denial", Aws::S3::S3Errors::ACCESS_DENIED); +#else + throw DB::Exception(DB::ErrorCodes::ABORTED, "store refusals need the S3 client"); +#endif + } + if (outage && outage()) + { + outage_writes.fetch_add(1, std::memory_order_relaxed); + if (on_outage_write) + on_outage_write(); + throw Poco::TimeoutException("injected mount PUT outage"); + } const Fault current = std::exchange(fault, Fault::None); if (current == Fault::BlockThenDelegate || current == Fault::BlockThenThrow) { @@ -2075,8 +2152,8 @@ class RuntimeUnderTest [this] { return runtime.fenceGeneration(); }, [this](uint64_t g, uint64_t needed) { return runtime.admit(g, needed); }, [this](uint64_t g) { runtime.checkFenceOrThrow(g); }}) - , farewell(backend, DB::Cas::Fence::open()) - , runtime(backend, mount, farewell, std::forward(args)...) + , lease(backend, DB::Cas::Fence::open()) + , runtime(backend, mount, lease, std::forward(args)...) { /// What the request engine reserves per attempt is the BACKEND's attempt timeout, not the /// budget field alone; every construction of this holder pairs the two via `runtimeRenewBudget`, @@ -2087,7 +2164,16 @@ class RuntimeUnderTest /// that injects one MUST inject the other, or `Retry::untilLeaseSafe` compares a synthetic /// deadline against real boottime, finds it long past, and refuses every request unsent. mount.setNowFnForTest([this] { return runtime.bootMsNow(); }); - farewell.setNowFnForTest([this] { return runtime.bootMsNow(); }); + lease.setNowFnForTest([this] { return runtime.bootMsNow(); }); + /// The lease plane's production sleep, so a stop or a park request ends a renewal's pause. + lease.setSleepFnForTest([this](uint64_t ms) { runtime.sleepInterruptibly(ms); }); + } + + /// The retry wait of the mount and lease planes. Call before the workers start. + void setRetrySleepForTest(const std::function & sleep_fn) + { + mount.setSleepFnForTest(sleep_fn); + lease.setSleepFnForTest(sleep_fn); } /// The workers are joined HERE, not only by the tests that assert on teardown: `CasMountRuntime` @@ -2109,7 +2195,7 @@ class RuntimeUnderTest private: DB::Cas::CasRequests mount; - DB::Cas::CasRequests farewell; + DB::Cas::CasRequests lease; CasMountRuntime runtime; }; @@ -2533,9 +2619,11 @@ TEST(CASMountOpenWaits, UncleanOpenPaysOnlyTheObservationWindow) })); ASSERT_TRUE(store); - /// The token-stability observation window is paid in full, pinned to the exact configured - /// formula (`mountObservationThresholdMs`): threshold_ms = ttl_ms + ttl_ms/20 + poll_interval_ms - /// = 500 + 25 + 50 = 575 ms, where poll_interval_ms = max(1, mount_renew_period/2) = 50 ms. The + /// The token-stability observation window is paid in full, pinned to the exact + /// formula (`mountObservationThresholdMs`). The predecessor declares the same lease, so own and + /// declared coincide: + /// threshold_ms = ttl_ms + ttl_ms/20 + poll_interval_ms = 500 + 25 + 50 = 575 ms, where + /// poll_interval_ms = max(1, mount_renew_period/2) = 50 ms. The /// loop only re-checks the threshold between polls, so the observed wait rounds UP to the next /// whole poll: ceil(575 / 50) * 50 = 600 ms, i.e. exactly 12 polls of 50 ms each -- because this /// predecessor's death was never certified, only observed. @@ -3820,7 +3908,7 @@ TEST(CASPoolRemount, NaturalTerminalTransitionMakesBothPersistentWorkersSelfExit "test", sink, runtimeRenewBudget(), [&] { if (terminal == PoolLifecycle::IdentityLost) - runtime_ptr->enterIdentityLost(); + runtime_ptr->enterIdentityLost("forced by the test"); else runtime_ptr->enterVanished(PoolLifecycle::VanishedReplaced, "injected natural replacement"); transitioned.arriveAndWait(); @@ -3926,7 +4014,7 @@ TEST(CASPoolRemount, ParkedRenewalCannotMissNaturalTerminalPublication) { parked_predicate_sampled_false.wait(); if (terminal == PoolLifecycle::IdentityLost) - runtime_ptr->enterIdentityLost(); + runtime_ptr->enterIdentityLost("forced by the test"); else runtime_ptr->enterVanished(PoolLifecycle::VanishedReplaced, "injected parked-wait replacement"); /// Before the fix, terminal publication does not wait for `driver_mutex`, so it reaches @@ -4150,12 +4238,8 @@ TEST(CASPoolRemount, TerminalDepositionDoesNotTouchRenewerAfterReplacement) runtime.installRenewer(uuid, 1, [&] { return wall_ms; }); const uint64_t anchor = runtime.startRenewer(); runtime.armMountFence(uuid, 1, anchor + 1000); - backend->fault = RuntimeRenewBackend::Fault::ThrowBefore; - /// Expire the lease from inside the attempt. The fault alone no longer ends a renewal: the engine - /// settles the ambiguity by reading and then reissues, and the reissue commits. With the clock past - /// the deadline the renewal was admitted under, neither the settling read nor the reissue is - /// admitted, so the renewal ends terminal -- which is what this test deposits. - backend->before_throw = [&, deadline = anchor + 1000] { boot_ms = deadline; }; + /// A definitive answer ends the worker's renewal; a transient fault would only be retried. + fenceOutMount(*backend, layout.mountKey("test")); runtime.startBackgroundWorkers(std::chrono::milliseconds(0)); terminal_deposited.waitUntilArrived(); EXPECT_TRUE(replaced.load(std::memory_order_acquire)); @@ -4240,11 +4324,8 @@ TEST(CASPoolRemount, ImmediatePostRemountRenewalFailureIsNotDropped) runtime_ptr->armMountFence(uuid, 2, fresh_anchor + 10'000); runtime_ptr->noteRemounted(); boot_ms = 2'000; - backend->fault = RuntimeRenewBackend::Fault::ThrowBefore; - /// Expire the fresh lease from inside the attempt, so the ambiguity can be neither - /// settled by a read nor reissued: otherwise the engine reissues and the renewal - /// commits, and there is no dropped failure to catch up on. - backend->before_throw = [&, deadline = fresh_anchor + 10'000] { boot_ms = deadline; }; + /// A definitive answer ends the worker's renewal; a transient fault would only be retried. + fenceOutMount(*backend, layout.mountKey("test")); first.arriveAndWait(); return true; } @@ -4269,6 +4350,418 @@ TEST(CASPoolRemount, ImmediatePostRemountRenewalFailureIsNotDropped) runtime.finishTeardown(false); } +namespace +{ +/// A mount slot claimed at epoch 1, a runtime renewing it in the background on an injected boot clock, and +/// the fence armed at the claim's anchor plus the lease. Locals the `MountConfig` hooks capture must be +/// declared before this object, which joins the workers when it is destroyed. +class BackgroundRenewal +{ +public: + BackgroundRenewal(const String & layout_name, uint64_t ttl_ms_) + : layout(layout_name) + , ttl_ms(ttl_ms_) + { + if (claimMount(*DB::Cas::tests::OperationForTest(backend), layout, "test", uuid, 1, wall_ms, ttl_ms).kind + != MountClaimResult::Claimed) + throw DB::Exception(DB::ErrorCodes::ABORTED, "test fixture claim did not land"); + } + + /// `config` gets the lease, the background flag and the boot clock here. + CasMountRuntime & open(MountConfig config, std::function remount = [] { return false; }) + { + config.mount_lease_ttl_ms = std::chrono::milliseconds(ttl_ms); + config.background_watermark = true; + config.boot_ms_fn = [this] { return boot_ms.load(); }; + holder.emplace(backend, layout, std::move(config), "test", sink, runtimeRenewBudget(), std::move(remount)); + CasMountRuntime & opened = runtime(); + opened.installRenewer(uuid, 1, [this] { return wall_ms; }); + anchor = opened.startRenewer(); + opened.armMountFence(uuid, 1, anchor + ttl_ms); + return opened; + } + + /// Every retry pause of the mount and lease planes moves the boot clock instead of sleeping. + void advanceClockOnRetrySleep() { holder->setRetrySleepForTest([this](uint64_t ms) { boot_ms += ms; }); } + + CasMountRuntime & runtime() { return **holder; } + + void stop() + { + runtime().stopBackgroundWorkers(); + runtime().finishTeardown(false); + } + + std::shared_ptr backend = std::make_shared(); + const Layout layout; + const UInt128 uuid{1}; + const uint64_t ttl_ms; + uint64_t wall_ms = 1000; + std::atomic boot_ms{100}; + CasEventSink sink; + uint64_t anchor = 0; + std::optional holder; +}; +} + +/// A renewal of the old epoch whose result is consumed after a remount armed the new epoch does not +/// move the new epoch's deadline. +TEST(CASMountRuntime, AnOldEpochRenewalDoesNotMoveTheNewEpochDeadline) +{ + DB::Cas::tests::ManualBarrier finished; + DB::Cas::tests::ManualBarrier remounted; + DB::Cas::tests::ManualBarrier back_in_loop; + std::atomic commits{0}; + std::atomic loop_passes{0}; + BackgroundRenewal renewal("runtime-old-epoch-deadline", 10'000); + CasMountRuntime & runtime = renewal.open( + MountConfig{ + .renewal_before_driver_lock_hook_for_test = [&] + { + if (++loop_passes == 2) + back_in_loop.arriveAndWait(); + }, + .renewal_finished_hook_for_test = [&](const MountRenewResult & result) + { + if (result.outcome == MountRenewOutcome::Committed && ++commits == 1) + finished.arriveAndWait(); + }}, + [&] + { + fenceOutMount(*renewal.backend, renewal.layout.mountKey("test")); + const MountClaimResult fresh = claimMount( + *DB::Cas::tests::OperationForTest(renewal.backend), renewal.layout, "test", renewal.uuid, 2, + renewal.wall_ms, renewal.ttl_ms); + EXPECT_EQ(fresh.kind, MountClaimResult::Claimed); + renewal.runtime().installRenewer(renewal.uuid, 2, [&] { return renewal.wall_ms; }); + const uint64_t fresh_anchor = renewal.runtime().startRenewer(); + renewal.runtime().armMountFence(renewal.uuid, 2, fresh_anchor + 5 * renewal.ttl_ms); + renewal.runtime().noteRemounted(); + remounted.arriveAndWait(); + return true; + }); + /// The epoch-1 renewal is due at once and inside its lease. The epoch-2 renewer is not driven before + /// the check: the worker waits in its second loop pass. + const uint64_t renewal_start = renewal.anchor + 1'000; + renewal.boot_ms = renewal_start; + runtime.startBackgroundWorkers(std::chrono::milliseconds(1'000)); + + finished.waitUntilArrived(); + runtime.tripMountLost(); + runtime.scheduleRemount(); + remounted.waitUntilArrived(); + remounted.release(); + finished.release(); + back_in_loop.waitUntilArrived(); + + /// Past the old renewal's deadline (its start plus one lease), before the new epoch's. + renewal.boot_ms = renewal_start + 2 * renewal.ttl_ms; + EXPECT_TRUE(runtime.mayMutate()) << "the old epoch's renewal moved the new epoch's deadline"; + back_in_loop.release(); + renewal.stop(); +} + +/// A mount PUT outage three leases long is a pause: the renewal keeps its tuple, its late success leaves +/// writes closed and starts the next renewal at once, that one opens them under the same fence +/// generation, and no remount is requested. +TEST(CASMountRuntime, ALongOutageIsAPauseUnderTheSameEpoch) +{ + std::atomic remount_calls{0}; + std::vector commits; + std::vector boot_at_commit; + std::vector may_mutate_after_commit; + DB::Cas::tests::ManualBarrier second_commit; + BackgroundRenewal renewal("until-definitive-long-outage", 1000); + CasMountRuntime & runtime = renewal.open( + MountConfig{ + .renewal_finished_hook_for_test = [&](const MountRenewResult & result) + { + if (result.outcome != MountRenewOutcome::Committed || commits.size() >= 2) + return; + commits.push_back(result); + boot_at_commit.push_back(renewal.boot_ms.load()); + may_mutate_after_commit.push_back(renewal.runtime().mayMutate()); + if (commits.size() == 2) + second_commit.arriveAndWait(); + }}, + [&] + { + ++remount_calls; + return false; + }); + const uint64_t generation = runtime.fenceGeneration(); + renewal.advanceClockOnRetrySleep(); + /// The first renewal is due at once and meets three lease lengths of outage. + const uint64_t anchor = renewal.anchor; + renewal.boot_ms = anchor + 500; + renewal.backend->outage = [&renewal, anchor] { return renewal.boot_ms.load() < anchor + 3'500; }; + runtime.startBackgroundWorkers(std::chrono::milliseconds(500)); + + second_commit.waitUntilArrived(); + EXPECT_EQ(commits[0].attempt_start_boot_ms, anchor + 500) << "the start of the logical renewal, not of its last attempt"; + EXPECT_GT(commits[0].attempts_sent, 2u); + EXPECT_GE(boot_at_commit[0], anchor + 3'500); + EXPECT_FALSE(may_mutate_after_commit[0]) << "a success whose start plus lease is past opens no write"; + EXPECT_EQ(commits[1].attempt_start_boot_ms, boot_at_commit[0]) << "the next renewal starts at once"; + EXPECT_TRUE(may_mutate_after_commit[1]); + EXPECT_EQ(runtime.fenceGeneration(), generation) << "a pause neither trips nor re-arms the fence"; + EXPECT_EQ(runtime.lifecycle(), PoolLifecycle::Live); + EXPECT_EQ(remount_calls.load(), 0u); + second_commit.release(); + renewal.stop(); +} + +/// A renewal that commits after its lease expired reports a reason that is true for a late commit too. +TEST(CASMountRuntime, ALateCommitReportsARecoveredRenewalWithATruthfulReason) +{ + std::atomic commits{0}; + std::atomic may_mutate_at_commit{true}; + std::optional recovered; + DB::Cas::tests::ManualBarrier recovered_reported; + BackgroundRenewal renewal("until-definitive-late-commit-reason", 1000); + renewal.sink = [&](CasEvent event) + { + if (event.type != CasEventType::WatermarkRenew || event.outcome != "recovered" || recovered) + return; + recovered = std::move(event); + recovered_reported.arriveAndWait(); + }; + CasMountRuntime & runtime = renewal.open(MountConfig{ + .renewal_finished_hook_for_test = [&](const MountRenewResult & result) + { + if (result.outcome == MountRenewOutcome::Committed && commits.fetch_add(1) == 0) + may_mutate_at_commit = renewal.runtime().mayMutate(); + }}); + renewal.advanceClockOnRetrySleep(); + const uint64_t anchor = renewal.anchor; + renewal.boot_ms = anchor + 500; + renewal.backend->outage = [&renewal, anchor] { return renewal.boot_ms.load() < anchor + 3'500; }; + runtime.startBackgroundWorkers(std::chrono::milliseconds(500)); + + recovered_reported.waitUntilArrived(); + EXPECT_FALSE(may_mutate_at_commit.load()) << "the commit came after the lease expired"; + ASSERT_TRUE(recovered.has_value()); + EXPECT_EQ(recovered->reason, "CAS mount renewal committed after a retry or a settling read"); + recovered_reported.release(); + renewal.stop(); +} + +/// A remount request ends a renewal that is retrying past its lease, at its next admission. +TEST(CASMountRuntime, ARemountRequestEndsARenewalRetryingPastTheLease) +{ + std::atomic held{false}; + std::atomic writes_at_remount{0}; + DB::Cas::tests::ManualBarrier holding; + DB::Cas::tests::ManualBarrier remount_entered; + BackgroundRenewal renewal("until-definitive-park", 1000); + CasMountRuntime & runtime = renewal.open(MountConfig{}, [&] + { + writes_at_remount = renewal.backend->outage_writes.load(); + remount_entered.arriveAndWait(); + return false; + }); + /// Five lease lengths on: a renewal bounded by its lease ended long before. + const uint64_t past_the_lease = renewal.anchor + 5'000; + renewal.advanceClockOnRetrySleep(); + renewal.backend->on_outage_write = [&renewal, &held, &holding, past_the_lease] + { + if (renewal.boot_ms.load() >= past_the_lease && !held.exchange(true)) + holding.arriveAndWait(); + }; + renewal.backend->outage = [] { return true; }; + runtime.startBackgroundWorkers(std::chrono::milliseconds(0)); + + holding.waitUntilArrived(); + const uint64_t writes_at_park = renewal.backend->outage_writes.load(); + runtime.scheduleRemount(); + EXPECT_EQ(runtime.renewalDriverStateForTest(), RenewalDriverState::ParkRequested); + holding.release(); + remount_entered.waitUntilArrived(); + EXPECT_EQ(writes_at_remount.load(), writes_at_park) << "no request starts after the remount request"; + remount_entered.release(); + renewal.stop(); +} + +/// The decommission tool's renewal stays bounded: under a permanent outage it ends inside its lease and +/// fences the mount instead of retrying past it. +TEST(CASMountRuntime, ABoundedBackgroundRenewalEndsInsideTheLease) +{ + DB::Cas::tests::ManualBarrier remount_entered; + BackgroundRenewal renewal("bounded-background-renewal", 1000); + CasMountRuntime & runtime = renewal.open(MountConfig{.bounded_background_renewal = true}, [&] + { + remount_entered.arriveAndWait(); + return false; + }); + renewal.advanceClockOnRetrySleep(); + renewal.boot_ms = renewal.anchor + 500; + renewal.backend->outage = [] { return true; }; + runtime.startBackgroundWorkers(std::chrono::milliseconds(500)); + + remount_entered.waitUntilArrived(); + EXPECT_FALSE(runtime.mayMutate()); + EXPECT_LT(renewal.boot_ms.load(), renewal.anchor + renewal.ttl_ms) << "the renewal outlived its lease"; + remount_entered.release(); + renewal.stop(); +} + +#if USE_AWS_S3 +/// A stop during the wait that follows a store refusal ends it at once, not one renewal period later, and +/// ends the series as cancelled: the farewell still hands the slot back. +TEST(CASMountRuntime, AStopDuringTheRefusalWaitEndsItAndKeepsTheFarewell) +{ + constexpr uint64_t period_ms = 30'000; + std::atomic first_write{true}; + DB::Cas::tests::ManualBarrier denied; + BackgroundRenewal renewal("until-definitive-refusal-stop", 2 * period_ms); + CasMountRuntime & runtime = renewal.open(MountConfig{}); + renewal.backend->deny_writes = 1; + renewal.backend->on_guarded_write = [&] + { + if (first_write.exchange(false)) + denied.arriveAndWait(); + }; + renewal.boot_ms = renewal.anchor + period_ms; + runtime.startBackgroundWorkers(std::chrono::milliseconds(period_ms)); + + denied.waitUntilArrived(); + denied.release(); + const auto stop_started = std::chrono::steady_clock::now(); + runtime.stopBackgroundWorkers(); + const auto stop_ms = std::chrono::duration_cast(std::chrono::steady_clock::now() - stop_started); + EXPECT_LT(stop_ms.count(), 10'000) << "the stop waited out the renewal period"; + runtime.finishTeardown(/*drained=*/true); + const auto left = (*DB::Cas::tests::OperationForTest(renewal.backend)).read(renewal.layout.mountKey("test"), Retry::standard()); + ASSERT_TRUE(left.has_value()); + EXPECT_EQ(decodeMountLease(left->bytes).min_active_build_sequence, std::numeric_limits::max()) + << "the farewell did not run"; +} + +/// A remount request during the wait that follows a store refusal ends it at once. +TEST(CASMountRuntime, AParkRequestEndsTheRefusalWaitAtOnce) +{ + constexpr uint64_t period_ms = 30'000; + std::atomic first_write{true}; + DB::Cas::tests::ManualBarrier denied; + DB::Cas::tests::ManualBarrier remount_entered; + BackgroundRenewal renewal("until-definitive-refusal-park", 2 * period_ms); + CasMountRuntime & runtime = renewal.open(MountConfig{}, [&] + { + remount_entered.arriveAndWait(); + return false; + }); + renewal.backend->deny_writes = std::numeric_limits::max(); + renewal.backend->on_guarded_write = [&] + { + if (first_write.exchange(false)) + denied.arriveAndWait(); + }; + renewal.boot_ms = renewal.anchor + period_ms; + runtime.startBackgroundWorkers(std::chrono::milliseconds(period_ms)); + + denied.waitUntilArrived(); + denied.release(); + const auto park_requested = std::chrono::steady_clock::now(); + runtime.scheduleRemount(); + remount_entered.waitUntilArrived(); + const auto park_ms = std::chrono::duration_cast(std::chrono::steady_clock::now() - park_requested); + EXPECT_LT(park_ms.count(), 10'000) << "the park request waited out the renewal period"; + remount_entered.release(); + renewal.stop(); +} + +/// After each store refusal the worker waits one renewal period before its next engine call. +TEST(CASMountRuntime, ARefusalWaitsOneRenewalPeriod) +{ + constexpr uint64_t period_ms = 700; + std::vector write_at; + std::atomic committed_once{false}; + DB::Cas::tests::ManualBarrier committed; + BackgroundRenewal renewal("until-definitive-refusal-period", 10'000); + CasMountRuntime & runtime = renewal.open(MountConfig{ + .renewal_finished_hook_for_test = [&](const MountRenewResult & result) + { + if (result.outcome == MountRenewOutcome::Committed && !committed_once.exchange(true)) + committed.arriveAndWait(); + }}); + renewal.advanceClockOnRetrySleep(); + renewal.backend->deny_writes = 2; + renewal.backend->on_guarded_write = [&] { write_at.push_back(renewal.boot_ms.load()); }; + renewal.boot_ms = renewal.anchor + period_ms; + runtime.startBackgroundWorkers(std::chrono::milliseconds(period_ms)); + + committed.waitUntilArrived(); + ASSERT_EQ(write_at.size(), 3u); + EXPECT_EQ(write_at[1] - write_at[0], period_ms); + EXPECT_EQ(write_at[2] - write_at[1], period_ms); + committed.release(); + renewal.stop(); +} +#endif + +/// One `WARNING` per pause, written at the first attempt after writes paused, and the pause is visible +/// from the moment writes are refused: the lease less one write's reservation and the safety margin. +TEST(CASMountRuntime, AWritePauseIsLoggedOnceAndReported) +{ + ScopedRemountLogCapture logs; + std::vector> paused_at_write; + std::atomic commits{0}; + String log_at_second_commit; + DB::Cas::tests::ManualBarrier second_commit; + BackgroundRenewal renewal("until-definitive-pause-log", 1000); + CasMountRuntime & runtime = renewal.open(MountConfig{ + .renewal_finished_hook_for_test = [&](const MountRenewResult & result) + { + if (result.outcome == MountRenewOutcome::Committed && ++commits == 2) + { + log_at_second_commit = logs.captured(); + second_commit.arriveAndWait(); + } + }}); + renewal.advanceClockOnRetrySleep(); + const uint64_t anchor = renewal.anchor; + renewal.boot_ms = anchor + 500; + renewal.backend->on_outage_write = [&] { paused_at_write.push_back(renewal.runtime().writesPausedSinceBootMs()); }; + renewal.backend->outage = [&renewal, anchor] { return renewal.boot_ms.load() < anchor + 3'500; }; + runtime.startBackgroundWorkers(std::chrono::milliseconds(500)); + + second_commit.waitUntilArrived(); + const String prefix = "CAS mount lease of 'test' under writer_epoch "; + size_t warnings = 0; + for (size_t at = log_at_second_commit.find(prefix); at != String::npos; at = log_at_second_commit.find(prefix, at + 1)) + ++warnings; + EXPECT_EQ(warnings, 1u) << log_at_second_commit; + EXPECT_NE(log_at_second_commit.find("Last renewal error: PUT "), String::npos) << log_at_second_commit; + ASSERT_GE(paused_at_write.size(), 3u); + EXPECT_FALSE(paused_at_write.front().has_value()) << "the first attempt runs while writes are admitted"; + const CasRequestBudget budget = runtimeRenewBudget(); + EXPECT_EQ(paused_at_write.back(), + std::optional(anchor + 1000 - 2 * budget.attemptEnvelopeMs() - budget.lease_safety_margin_ms)); + EXPECT_FALSE(runtime.writesPausedSinceBootMs().has_value()) << "a renewal with lease left ends the pause"; + second_commit.release(); + renewal.stop(); +} + +/// A clean shutdown while writes are paused leaves the slot for observation: the farewell needs lease room +/// that the pause no longer has. +TEST(CASMountRuntime, ACleanShutdownDuringAPauseLeavesTheSlotForObservation) +{ + ScopedRemountLogCapture logs; + BackgroundRenewal renewal("pause-shutdown", 1000); + CasMountRuntime & runtime = renewal.open(MountConfig{}); + renewal.boot_ms = renewal.anchor + renewal.ttl_ms; + ASSERT_TRUE(runtime.writesPausedSinceBootMs().has_value()); + + runtime.finishTeardown(/*drained=*/true); + + const String captured = logs.captured(); + EXPECT_EQ(captured.find("release during Pool teardown failed"), String::npos) << captured; + EXPECT_NE(captured.find("the slot is left for observation"), String::npos) << captured; + const auto left = (*DB::Cas::tests::OperationForTest(renewal.backend)).read(renewal.layout.mountKey("test"), Retry::standard()); + ASSERT_TRUE(left.has_value()); + EXPECT_NE(decodeMountLease(left->bytes).min_active_build_sequence, std::numeric_limits::max()); +} + TEST(CASPoolRemount, StaleRemountAnchorPerformsParkedRedo) { auto backend = std::make_shared(); @@ -4672,10 +5165,8 @@ TEST(CASPool, DeterministicWorkerFailureFencesWithoutWaitingForCadence) runtime.installRenewer(uuid, 1, [&] { return wall_ms; }); const uint64_t anchor = runtime.startRenewer(); runtime.armMountFence(uuid, 1, anchor + 1000); - backend->fault = RuntimeRenewBackend::Fault::ThrowBefore; - /// Expire the lease from inside the attempt, so the ambiguity can be neither settled by a read nor - /// reissued: without that the engine reissues and the renewal commits, and this worker never fences. - backend->before_throw = [&, deadline = anchor + 1000] { boot_ms = deadline; }; + /// A definitive answer ends the worker's renewal; a transient fault would only be retried. + fenceOutMount(*backend, layout.mountKey("test")); runtime.startBackgroundWorkers(std::chrono::milliseconds(0)); remount_entered.waitUntilArrived(); EXPECT_FALSE(runtime.mayMutate()); diff --git a/src/Disks/tests/gtest_cas_probe.cpp b/src/Disks/tests/gtest_cas_probe.cpp index 6b9da190dcade..136f36370547b 100644 --- a/src/Disks/tests/gtest_cas_probe.cpp +++ b/src/Disks/tests/gtest_cas_probe.cpp @@ -48,7 +48,17 @@ TEST(CASProbe, FailsClosedOnDeleteMarkers) b->setSimulateDeleteMarkers(true); // versioning enabled on the prefix auto requests = makeRequests(*b); auto op = requests.admit(); - EXPECT_THROW(runCapabilityProbe(op, "p/.cas_probe"), DB::Exception); + String message; + try + { + runCapabilityProbe(op, "p/.cas_probe"); + } + catch (const DB::Exception & e) + { + message = e.message(); + } + EXPECT_NE(message.find("does not make a versioned bucket usable"), String::npos) << message; + EXPECT_EQ(message.find("has no override"), String::npos) << message; } TEST(CASProbe, PassesOnEmulatedLocal) diff --git a/src/Disks/tests/gtest_cas_protocol_scenarios.cpp b/src/Disks/tests/gtest_cas_protocol_scenarios.cpp index f33018415c0af..73c73b243ecb6 100644 --- a/src/Disks/tests/gtest_cas_protocol_scenarios.cpp +++ b/src/Disks/tests/gtest_cas_protocol_scenarios.cpp @@ -50,6 +50,7 @@ using DB::Cas::tests::loadMetaForTest; using DB::Cas::tests::streamingHexOf; using DB::Cas::tests::u128Of; using DB::Cas::tests::writeBlobRaw; +using DB::Cas::tests::writeMetaClean; namespace { @@ -190,8 +191,9 @@ TEST(CASProtocol, RevalidateReObservesStaleTokenKeepsWhenUnchanged) auto s = openPool(b); const RootNamespace ns{"srv1/tbl"}; - /// X pre-exists out-of-band; the build dedup-adopts it via putBlob (records the current token t0). + /// X and its `Clean` marker pre-exist out-of-band; the build dedup-adopts X via putBlob (token t0). writeBlobRaw(*b, s->layout(), "payload-X", s->poolMeta().blob_header_len, s->poolMeta().pool_id); + writeMetaClean(*b, s->layout(), u128Of("payload-X"), 9); const String blob_key = s->layout().blobKey(idOf("payload-X")); const Etag t0 = currentIncarnation(*b, blob_key); @@ -199,7 +201,7 @@ TEST(CASProtocol, RevalidateReObservesStaleTokenKeepsWhenUnchanged) auto build = startBuildFor(s, ns, "part_1"); const ManifestId id = build->stageManifest({blobEntry("data.bin", "payload-X")}); build->precommitAdd(ns, "part_1", id); - build->putBlob(idOf("payload-X"), BlobSource::fromString("payload-X")); /// dedup → adopts t0 + build->putBlob(idOf("payload-X"), BlobSource::fromString("payload-X")); /// adopts t0 on its `Clean` marker /// GC advanced the round to 1 with an EMPTY retired set; fence to 1. X is NOT condemned and its /// token is unchanged. @@ -224,6 +226,7 @@ TEST(CASProtocol, RevalidateReObservesStaleTokenAdoptsWhenDisplaced) const RootNamespace ns{"srv1/tbl"}; writeBlobRaw(*b, s->layout(), "payload-X", s->poolMeta().blob_header_len, s->poolMeta().pool_id); + writeMetaClean(*b, s->layout(), u128Of("payload-X"), 9); const String blob_key = s->layout().blobKey(idOf("payload-X")); const Etag t0 = currentIncarnation(*b, blob_key); @@ -231,7 +234,7 @@ TEST(CASProtocol, RevalidateReObservesStaleTokenAdoptsWhenDisplaced) /// Wiring order (EDGE-BEFORE-OBSERVE): stageManifest -> precommitAdd -> putBlob. const ManifestId id = build->stageManifest({blobEntry("data.bin", "payload-X")}); build->precommitAdd(ns, "part_1", id); - build->putBlob(idOf("payload-X"), BlobSource::fromString("payload-X")); /// dedup → adopts t0 + build->putBlob(idOf("payload-X"), BlobSource::fromString("payload-X")); /// adopts t0 on its `Clean` marker /// Another writer displaces X out-of-band ⇒ a new current token t1 (same payload, fresh tag). const Etag t1 = displaceBlobToken(*b, s->layout(), idOf("payload-X")); @@ -269,6 +272,7 @@ TEST(CASProtocol, RevalidateAdoptsLiveTokenWhenOnlyPhantomCondemnedAtDifferentTo { auto s0 = Pool::open(b, PoolConfig{.pool_prefix = "p", .server_root_id = "test"}); writeBlobRaw(*b, s0->layout(), "payload-X", s0->poolMeta().blob_header_len, s0->poolMeta().pool_id); + writeMetaClean(*b, s0->layout(), u128Of("payload-X"), 9); } const String blob_key = layout.blobKey(idOf("payload-X")); const Etag t0 = currentIncarnation(*b, blob_key); @@ -285,7 +289,7 @@ TEST(CASProtocol, RevalidateAdoptsLiveTokenWhenOnlyPhantomCondemnedAtDifferentTo auto build = startBuildFor(s, ns, "part_1"); const ManifestId id = build->stageManifest({blobEntry("data.bin", "payload-X")}); build->precommitAdd(ns, "part_1", id); - build->putBlob(idOf("payload-X"), BlobSource::fromString("payload-X")); /// dedup → adopts t0 + build->putBlob(idOf("payload-X"), BlobSource::fromString("payload-X")); /// adopts t0 on its `Clean` marker /// promote: the materialized leaf is edge-protected (not re-observed) ⇒ commit. Lands. t0 untouched. build->promote(ns, "part_1", build->buildId(), id); @@ -480,6 +484,7 @@ TEST(CASProtocol, DisplacedToLiveTokenCommitsAtCurrentIncarnation) const RootNamespace ns{"srv1/tbl"}; writeBlobRaw(*b, s->layout(), "payload-X", s->poolMeta().blob_header_len, s->poolMeta().pool_id); + writeMetaClean(*b, s->layout(), u128Of("payload-X"), 9); const String blob_key = s->layout().blobKey(idOf("payload-X")); const Etag t0 = currentIncarnation(*b, blob_key); @@ -487,7 +492,7 @@ TEST(CASProtocol, DisplacedToLiveTokenCommitsAtCurrentIncarnation) /// Wiring order (EDGE-BEFORE-OBSERVE): stageManifest -> precommitAdd -> putBlob. const ManifestId id = build->stageManifest({blobEntry("data.bin", "payload-X")}); build->precommitAdd(ns, "part_1", id); - build->putBlob(idOf("payload-X"), BlobSource::fromString("payload-X")); /// dedup → adopts t0 + build->putBlob(idOf("payload-X"), BlobSource::fromString("payload-X")); /// adopts t0 on its `Clean` marker /// Another writer displaces X to t1 (uncondemned) before our gate runs. const Etag t1 = displaceBlobToken(*b, s->layout(), idOf("payload-X")); diff --git a/src/Disks/tests/gtest_cas_ref_ckpt.cpp b/src/Disks/tests/gtest_cas_ref_ckpt.cpp index ddbf9f33c26f1..f68c0f219b301 100644 --- a/src/Disks/tests/gtest_cas_ref_ckpt.cpp +++ b/src/Disks/tests/gtest_cas_ref_ckpt.cpp @@ -771,8 +771,22 @@ TEST(CASRefCheckpoint, AnExhaustedDeadlineUnderPersistentConflictThrowsRetryLate }; const RefCkpt advance{.life_epoch = std::nullopt, .committed_through = ID_1_2, .checkpoint_snapshot_id = ID_1_2, .last_epoch_seal = std::nullopt}; - expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] { publishCkpt(op, layout, life, advance); }); + String message; + expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] + { + try + { + publishCkpt(op, layout, life, advance); + } + catch (const DB::Exception & e) + { + message = e.message(); + throw; + } + }); backend->after_read = nullptr; + EXPECT_NE(message.find("policy deadline"), String::npos) << message; + EXPECT_EQ(message.find("contention"), String::npos) << message; EXPECT_FALSE(clock.sleeps.empty()) << "the loop must back off between attempts, not spin"; EXPECT_EQ(readCkptOrFail(setup, layout, life), base) << "no partial state: every attempt either " "committed the complete merged body or wrote nothing"; diff --git a/src/Disks/tests/gtest_cas_ref_install_safety.cpp b/src/Disks/tests/gtest_cas_ref_install_safety.cpp index dbc09c25a090b..b0b93bd4d4ec2 100644 --- a/src/Disks/tests/gtest_cas_ref_install_safety.cpp +++ b/src/Disks/tests/gtest_cas_ref_install_safety.cpp @@ -306,8 +306,10 @@ TEST(CASRefInstallSafety, UnresolvedAlwaysRecordsTheWedge) /// backend's `putIfAbsent`, so the request reached it), the single-attempt budget is then spent, and /// the lane wedges. The message must say so -- and must NOT say "no attempt was sent", which is the /// only shape allowed to skip the wedge. - EXPECT_NE(message.find("is UNCERTAIN"), String::npos) + EXPECT_NE(message.find("outcome is unknown"), String::npos) << "the reason must reach the wedge message rather than defaulting: " << message; + EXPECT_NE(message.find("a retry may duplicate it"), String::npos) << message; + EXPECT_EQ(message.find("nothing is durable"), String::npos) << message; EXPECT_EQ(message.find("BEFORE any request was sent"), String::npos) << "an ambiguous PUT is not a pre-attempt refusal: " << message; } @@ -350,6 +352,7 @@ TEST(CASRefInstallSafety, PreAttemptRefusalDoesNotWedgeTheLane) EXPECT_NE(message.find("refused BEFORE any request was sent"), String::npos) << "the caller must be told WHY, and this is the reason the no-wedge decision rests on: " << message; + EXPECT_NE(message.find("nothing is durable, so a retry cannot duplicate this transaction"), String::npos) << message; EXPECT_FALSE(store->refLaneWedgedForTest(ns)) << "nothing was sent, so nothing can be durable: there is no ambiguity for a wedge to resolve"; EXPECT_TRUE(store->wedgedKeyForTest(ns).empty()); diff --git a/src/Disks/tests/gtest_cas_ref_recovery_cas_walk.cpp b/src/Disks/tests/gtest_cas_ref_recovery_cas_walk.cpp index 5d92d37e9502a..8581a6d3b1ebc 100644 --- a/src/Disks/tests/gtest_cas_ref_recovery_cas_walk.cpp +++ b/src/Disks/tests/gtest_cas_ref_recovery_cas_walk.cpp @@ -404,7 +404,7 @@ void burnEpochsUpTo(const BackendPtr & backend, const Layout & layout, uint64_t CasRequests requests(backend, Fence::open()); CasOperation op = requests.admit(); for (uint64_t e = 1; e < target_live_epoch; ++e) - allocateWriterEpoch(op, layout, "test", EpochMintPolicy::NormalMount, 0, [] { return RefCatalog{}; }); + allocateWriterEpoch(op, layout, "test", [] { return RefCatalog{}; }); } /// One ordinary transaction at `id`, publishing `ref` (prepending the birth op when `birth`). diff --git a/src/Disks/tests/gtest_cas_ref_wedge_every_attempt.cpp b/src/Disks/tests/gtest_cas_ref_wedge_every_attempt.cpp index c0d66fad0a7a6..e05710357126d 100644 --- a/src/Disks/tests/gtest_cas_ref_wedge_every_attempt.cpp +++ b/src/Disks/tests/gtest_cas_ref_wedge_every_attempt.cpp @@ -490,7 +490,7 @@ void wedgeLaneOverADurableObject(VirtualRetryClock & clock, WedgeTestBackend & b backend.landed_ack_lost_latched = true; backend.fail_get_substr = logPrefix(store, ns); backend.fail_get_latched = true; - expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] { store->dropRef(ns, ref); }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "outcome is unknown", [&] { store->dropRef(ns, ref); }); backend.landed_ack_lost_latched = false; backend.landed_ack_lost_substr.clear(); backend.fail_get_latched = false; @@ -512,7 +512,7 @@ void wedgeLaneOnUnresolvedAppend(VirtualRetryClock & clock, WedgeTestBackend & b backend.ambiguous_substr = logPrefix(store, ns); backend.ambiguous_latched = true; - expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, drive); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "outcome is unknown", drive); backend.ambiguous_latched = false; backend.ambiguous_substr.clear(); @@ -593,7 +593,8 @@ TEST(CASRefWedgeEveryAttempt, DurableCreatedWedgeNeedsRecoveryWhenItsFrontierCan backend->fail_cas_latched = true; const size_t pauses_before = clock->pauseCount(); - expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] { store->dropRef(ns, "y"); }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, + "is durable but was not acknowledged", [&] { store->dropRef(ns, "y"); }); backend->fail_cas_latched = false; backend->fail_cas_substr.clear(); @@ -672,7 +673,8 @@ TEST(CASRefWedgeEveryAttempt, RetiredLifeRefusesWedgeRetryBeforeAnyRequestOrAdop retry.join(); store->setWedgeBeforeSlotOccupyHookForTest(nullptr); - EXPECT_TRUE(retry_error); + ASSERT_TRUE(retry_error); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "outcome is unknown", [&] { std::rethrow_exception(retry_error); }); EXPECT_EQ(backend->putCount(wedged_key), 0u) << "retirement must refuse before the retry send"; EXPECT_EQ(backend->getCount(wedged_key), 0u) << "a refused retry needs no occupant resolution read"; EXPECT_FALSE(readObj(*backend, wedged_key)) << "the predecessor wedge was adopted or made durable"; @@ -734,7 +736,8 @@ TEST(CASRefWedgeEveryAttempt, DefiniteRefusalOfARetryAttemptKeepsTheLaneWedged) /// The retry's own create is definitively refused. backend->definite_substr = logPrefix(store, ns); backend->definite_count = 1; - EXPECT_ANY_THROW(store->dropRef(ns, "y")); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "outcome is unknown", + [&] { store->dropRef(ns, "y"); }); EXPECT_TRUE(store->refLaneWedgedForTest(ns)) << "a definite refusal AFTER an ambiguous attempt must not unwedge"; EXPECT_EQ(store->wedgedKeyForTest(ns), wedged_key) << "the SAME wedge, not a fresh one"; @@ -900,7 +903,8 @@ TEST(CASRefWedgeEveryAttempt, SuccessorSealAtTheWedgedKeyRejectsConclusivelyAndS /// The next caller's resolution meets the seal. Its own items fail -- permanently, not "retry /// later": nothing about this lane's epoch will ever accept a write again. - expectThrowsCode(DB::ErrorCodes::INVALID_STATE, [&] { store->dropRef(ns, "y"); }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::INVALID_STATE, "nothing is durable", + [&] { store->dropRef(ns, "y"); }); EXPECT_FALSE(store->refLaneWedgedForTest(ns)) << "a conclusive rejection clears the wedge"; EXPECT_TRUE(store->resolveRef(ns, "x").has_value()) << "the rejected transaction was never applied"; @@ -1020,7 +1024,8 @@ TEST(CASRefWedgeEveryAttempt, ForeignNonSealOccupantIsCorruptedDataAndSchedulesA /// Something that is neither our bytes nor a seal occupies the slot. ASSERT_TRUE(std::holds_alternative(backend->putAsSuccessor(wedged_key, "not a ref-log object at all"))); - expectThrowsCode(DB::ErrorCodes::CORRUPTED_DATA, [&] { store->dropRef(ns, "y"); }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::CORRUPTED_DATA, "outcome is unknown", + [&] { store->dropRef(ns, "y"); }); EXPECT_FALSE(store->refLaneWedgedForTest(ns)); EXPECT_EQ(store->laneStateForTest(ns), RefLaneState::Faulted); @@ -1085,7 +1090,9 @@ TEST(CASRefWedgeEveryAttempt, RetryUnderAnOlderAdmissionGenerationSendsNothing) bumpFenceGeneration(store, epoch); ASSERT_NE(store->wedgedAdmittedGenerationForTest(ns), store->fenceGeneration()); - EXPECT_ANY_THROW(store->dropRef(ns, "y")); + /// Refused at admission, before the write reaches the queue: no wedge phrase, by design. + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "no longer admitted", + [&] { store->dropRef(ns, "y"); }); EXPECT_EQ(backend->putCount(wedged_key), puts_before) << "the retry must be refused pre-attempt: nothing may reach the store under a foreign generation"; @@ -1136,7 +1143,8 @@ TEST(CASRefWedgeEveryAttempt, ResultReleasedAfterAFenceBumpAndSuccessorSealIsIne /// And it must be the RETRY-SAFE class. A moved incarnation is usually a routine lease blip, and the /// storage layer classifies retry-safety on exactly `ABORTED || NETWORK_ERROR` — surfacing the fence /// check's own `INVALID_STATE` here would turn every blip into a hard failure for the caller. - expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] { std::rethrow_exception(caller_error); }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "outcome is unknown", + [&] { std::rethrow_exception(caller_error); }); EXPECT_TRUE(store->refLaneWedgedForTest(ns)) << "no unwedge"; EXPECT_EQ(store->tailSinceSnapshotCountForTest(ns), tail_before) << "no install"; EXPECT_EQ(store->lastEpochSealForTest(ns), std::nullopt) @@ -1401,7 +1409,8 @@ TEST(CASRefWedgeEveryAttempt, CreationCkptSurvivesWhenTheFirstNamespaceBirthName /// settling read proves absence and the conflict comes back naming no occupant at all. backend->refuse_precondition_substr = layout.refLogKey(DB::Cas::tests::fixture::fixtureLife(ns), genesis); - expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] { publishEmptyPart(store, ns, "x"); }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "outcome is unknown", + [&] { publishEmptyPart(store, ns, "x"); }); const auto ckpt_after = readObj(*backend, ckpt_key); ASSERT_TRUE(ckpt_after.has_value()); @@ -1467,7 +1476,8 @@ TEST(CASRefWedgeEveryAttempt, AppendSiteWedgesWhenTheSettlingReadNamesNoOccupant const uint64_t deferred_before = ProfileEvents::global_counters[ProfileEvents::CASRefAppendOccupantUnreadable]; const uint64_t wedged_before = ProfileEvents::global_counters[ProfileEvents::CASRefAppendWedged]; - expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] { store->dropRef(ns, "x"); }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "outcome is unknown", + [&] { store->dropRef(ns, "x"); }); EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASRefAppendOccupantUnreadable], deferred_before + 1) << "the deferral is the one quiet arm here -- it must be counted or a starved loud path is invisible"; @@ -1513,7 +1523,8 @@ TEST(CASRefWedgeEveryAttempt, AppendSiteWedgesWhenTheSettlingReadItselfIsRefused backend->refuse_read_after_precondition = true; const uint64_t deferred_before = ProfileEvents::global_counters[ProfileEvents::CASRefAppendOccupantUnreadable]; - expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] { store->dropRef(ns, "x"); }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "outcome is unknown", + [&] { store->dropRef(ns, "x"); }); EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASRefAppendOccupantUnreadable], deferred_before + 1); EXPECT_TRUE(store->mayMutate()); @@ -1661,7 +1672,8 @@ TEST(CASRefLane, RefusedReturnsTheAttemptToReadyAndDoesNotWedge) const uint64_t wedged_before = ProfileEvents::global_counters[ProfileEvents::CASRefAppendWedged]; const uint64_t definite_before = ProfileEvents::global_counters[ProfileEvents::CASRefAppendDefiniteFailure]; - expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] { store->dropRef(ns, "x"); }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, + "nothing is durable, so a retry cannot duplicate this transaction", [&] { store->dropRef(ns, "x"); }); EXPECT_FALSE(store->refLaneWedgedForTest(ns)) << "a proven refusal wrote nothing, so there is nothing for a wedge to resolve"; diff --git a/src/Disks/tests/gtest_cas_ref_writer.cpp b/src/Disks/tests/gtest_cas_ref_writer.cpp index fe5e08ce183a6..ddb79e6fa1c13 100644 --- a/src/Disks/tests/gtest_cas_ref_writer.cpp +++ b/src/Disks/tests/gtest_cas_ref_writer.cpp @@ -258,8 +258,7 @@ uint64_t allocateWriterEpochForTest(const BackendPtr & backend, const Layout & l { CasRequests requests(backend, Fence::open()); CasOperation op = requests.admit(); - return allocateWriterEpoch(op, layout, server_root_id, EpochMintPolicy::NormalMount, 0, - [] { return RefCatalog{}; }); + return allocateWriterEpoch(op, layout, server_root_id, [] { return RefCatalog{}; }); } /// A fixture enumeration on an open fence: the primitive `list` override in this file's test backend @@ -1559,7 +1558,8 @@ TEST(CASRefWriterAppendLane, CheckpointConflictAfterLogCommitRequiresRecoveryWit auto clock = VirtualRetryClock::installOn(store); backend->ckpt_conflict_key = ckpt_key; backend->ckpt_conflict_latched = true; - expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] { store->dropRef(ns, "x"); }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, + "is durable but was not acknowledged", [&] { store->dropRef(ns, "x"); }); backend->disarmFaults(); EXPECT_GT(clock->pauseCount(), 1u) @@ -1577,6 +1577,62 @@ TEST(CASRefWriterAppendLane, CheckpointConflictAfterLogCommitRequiresRecoveryWit << "no later id may be allocated above an unfrontiered durable transaction"; } +/// The same `_ckpt` failure after the ref-log commit means different things per step: a promoted part +/// can come back after a restart, a precommitted one is never published. The texts must differ. +TEST(CASRefWriterAppendLane, DurableUnconfirmedTextNamesPrecommitAndPromoteApart) +{ + auto backend = std::make_shared(); + auto store = openPool(backend); + const RootNamespace ns{"srv1/frontier-conflict-steps"}; + publishEmptyPart(store, ns, "x"); + const NamespaceLifeId life = lifeIfCatalogedForTest(backend, store->layout(), ns).value(); + const String ckpt_key = store->layout().refCkptKey(life); + auto clock = VirtualRetryClock::installOn(store); + + auto messageOf = [&](const std::function & step) + { + backend->ckpt_conflict_key = ckpt_key; + backend->ckpt_conflict_latched = true; + String message; + try + { + step(); + } + catch (const DB::Exception & e) + { + message = e.message(); + } + backend->disarmFaults(); + return message; + }; + + auto precommitted = startBuildFor(store, ns, "pre"); + const ManifestId pre_id = precommitted->stageManifest({}); + const String precommit_message = messageOf([&] { precommitted->precommitAdd(ns, "pre", pre_id); }); + precommitted.reset(); + + /// Recover the lane so the promote below reaches its own ref-log commit. + publishEmptyPart(store, ns, "between"); + + auto promoted = startBuildFor(store, ns, "prom"); + const ManifestId prom_id = promoted->stageManifest({}); + promoted->precommitAdd(ns, "prom", prom_id); + const String promote_message = messageOf([&] { promoted->promote(ns, "prom", promoted->buildId(), prom_id); }); + + /// A kind other than promote must not carry the promote-only note. + publishEmptyPart(store, ns, "between_again"); + const String drop_message = messageOf([&] { store->dropRef(ns, "x"); }); + EXPECT_NE(drop_message.find("is durable but was not acknowledged"), String::npos) << drop_message; + EXPECT_EQ(drop_message.find("promoted part returns"), String::npos) << drop_message; + + EXPECT_NE(precommit_message.find("never published"), String::npos) << precommit_message; + EXPECT_EQ(precommit_message.find("may still take effect"), String::npos) << precommit_message; + EXPECT_NE(promote_message.find("is durable but was not acknowledged"), String::npos) << promote_message; + EXPECT_NE(promote_message.find("may still take effect"), String::npos) << promote_message; + EXPECT_EQ(promote_message.find("restart"), String::npos) << promote_message; + EXPECT_EQ(promote_message.find("never published"), String::npos) << promote_message; +} + TEST(CASRefWriterAppendLane, FenceMovementAtCheckpointPublicationRequiresRecoveryWithoutInstall) { auto backend = std::make_shared(); @@ -1594,7 +1650,9 @@ TEST(CASRefWriterAppendLane, FenceMovementAtCheckpointPublicationRequiresRecover backend->ckpt_get_hook_key = ckpt_key; backend->ckpt_get_hook = [&] { store->tripMountLost(); }; - expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] { store->dropRef(ns, "x"); }); + SCOPE_EXIT({ backend->ckpt_get_hook = nullptr; }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, + "is durable but was not acknowledged", [&] { store->dropRef(ns, "x"); }); EXPECT_EQ(store->laneStateForTest(ns), RefLaneState::NeedsRecovery); EXPECT_EQ(store->tailSinceSnapshotCountForTest(ns), tail_before) diff --git a/src/Disks/tests/gtest_cas_request_observer.cpp b/src/Disks/tests/gtest_cas_request_observer.cpp new file mode 100644 index 0000000000000..3aa6565803278 --- /dev/null +++ b/src/Disks/tests/gtest_cas_request_observer.cpp @@ -0,0 +1,83 @@ +#include + +#include +#include +#include +#include + +#include + +#include + +#include +#include + +namespace DB::ErrorCodes +{ +extern const int BAD_ARGUMENTS; +extern const int NETWORK_ERROR; +} + +using namespace DB::Cas; + +namespace +{ + +/// One seeded key and an operation whose observer records every call as a line. +struct Fixture +{ + std::shared_ptr backend = std::make_shared(); + CasRequests requests{backend, Fence::open(), [] { return uint64_t{100}; }, [](uint64_t) {}}; + Etag etag = std::get(requests.admit().create("k", "v1", Retry::standard())).etag; + std::vector events; + CasOperation op = requests.admit(); + + Fixture() + { + op.observeRequests(RequestObserver{ + .on_write_attempt = [this](uint32_t attempts_sent) { events.push_back(fmt::format("attempt {}", attempts_sent)); }, + .on_failure = [this](std::string_view verb, const String & key, const std::exception & e) + { + events.push_back(describeRequestFailure(verb, key, e)); + }, + }); + } +}; + +} + +TEST(CASRequestObserver, WriteAttemptsAndFailuresAreReportedAsTheyHappen) +{ + Fixture f; + f.backend->failNextWriteWith("k", std::make_exception_ptr(Poco::TimeoutException("injected write timeout"))); + f.backend->failNextReadWith("k", std::make_exception_ptr(DB::Exception(DB::ErrorCodes::NETWORK_ERROR, "injected read failure"))); + + ASSERT_TRUE(std::holds_alternative(f.op.replace("k", "v2", f.etag, Retry::standard()))); + EXPECT_EQ(f.events, (std::vector{ + "attempt 1", + "PUT k: transport Timeout injected write timeout", + "GET k: transport NETWORK_ERROR injected read failure", + "attempt 2", + })); +} + +/// A local bug is not an answer from the store, so it is not published as the request's error. +TEST(CASRequestObserver, ALocalFailureIsNotReportedAsAStoreAnswer) +{ + Fixture f; + f.backend->failNextWriteWith("k", std::make_exception_ptr(DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "injected local failure"))); + + EXPECT_THROW((void)f.op.replace("k", "v2", f.etag, Retry::standard()), DB::Exception); + EXPECT_EQ(f.events, (std::vector{"attempt 1"})); +} + +/// A local failure of the resolve read is not published either, only the store answer before it. +TEST(CASRequestObserver, ALocalFailureOfTheResolveReadIsNotReported) +{ + Fixture f; + f.backend->failNextWriteWith("k", std::make_exception_ptr(Poco::TimeoutException("injected write timeout"))); + f.backend->failNextReadWith("k", std::make_exception_ptr(DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "injected local failure"))); + + EXPECT_THROW((void)f.op.replace("k", "v2", f.etag, Retry::standard()), DB::Exception); + EXPECT_EQ(f.events, (std::vector{"attempt 1", "PUT k: transport Timeout injected write timeout"})); +} diff --git a/src/Disks/tests/gtest_cas_requests.cpp b/src/Disks/tests/gtest_cas_requests.cpp index af812ef79483f..6472135ccab21 100644 --- a/src/Disks/tests/gtest_cas_requests.cpp +++ b/src/Disks/tests/gtest_cas_requests.cpp @@ -34,6 +34,7 @@ #include +#include #include #include #include @@ -223,6 +224,13 @@ TEST(CASWriteResult, OrThrowMapsEveryAlternative) .sent_any = false, .last_seen = NotObserved{}}; for (const GaveUp & gave_up : {deadline, unresolved, fence_lost}) expectThrowsCode(DB::ErrorCodes::NETWORK_ERROR, [&] { orThrow(WriteResult{gave_up}, "t"); }); + + GaveUp unsent_unresolved = unresolved; + unsent_unresolved.sent_any = false; + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "no write was sent", + [&] { orThrow(WriteResult{unsent_unresolved}, "t"); }); + DB::Cas::tests::expectThrowsCodeWithMessage(DB::ErrorCodes::NETWORK_ERROR, "the write is unresolved (sent", + [&] { orThrow(WriteResult{unresolved}, "t"); }); } TEST(CASFence, OpenFenceAdmitsEverythingAndNeverMoves) @@ -610,6 +618,51 @@ TEST(CASRetry, BindSaturatesAndLeavesAnEqualLeaseOffTheLeaseSource) EXPECT_TRUE(lease.lease_bound); } +TEST(CASRetrySpacing, UntilDefinitiveHasNoDeadlineAndSpacesItsAttempts) +{ + constexpr uint64_t largest = std::numeric_limits::max(); + const Retry policy = Retry::untilDefinitive(); + EXPECT_FALSE(policy.lease_deadline_ms.has_value()); + EXPECT_FALSE(policy.single_attempt); + EXPECT_EQ(policy.attempt_spacing_ms, std::optional(1'000)); + for (const uint64_t now : {uint64_t{0}, uint64_t{1'000'000}, largest - 1, largest}) + { + const Retry::Bound bound = policy.bind(now); + EXPECT_EQ(bound.deadline_ms, largest) << "now " << now; + EXPECT_FALSE(bound.lease_bound) << "now " << now; + } + EXPECT_FALSE(Retry::standard().attempt_spacing_ms.has_value()); + EXPECT_FALSE(Retry::untilLeaseSafe(2'000'000, 2'000).attempt_spacing_ms.has_value()); +} + +TEST(CASRetrySpacing, SpacedPauseWaitsOutTheRestOfTheDraw) +{ + EXPECT_EQ(Retry::spacedPause(1'000, 5'000, 5'000), 1'000u); + EXPECT_EQ(Retry::spacedPause(1'000, 5'000, 5'300), 700u); + EXPECT_EQ(Retry::spacedPause(1'000, 5'000, 6'000), 0u); + EXPECT_EQ(Retry::spacedPause(1'000, 5'000, 10'000), 0u); + EXPECT_EQ(Retry::spacedPause(1'000, 5'000, 4'000), 1'000u); +} + +TEST(CASRetrySpacing, DrawIsWithinAFifthOfTheSpacing) +{ + bool low = false; + bool high = false; + for (int i = 0; i < 2'000; ++i) + { + const uint64_t draw = Retry::drawSpacing(1'000); + ASSERT_GE(draw, 800u); + ASSERT_LE(draw, 1'200u); + low = low || draw < 850; + high = high || draw > 1'150; + } + /// The bounds alone cannot tell a draw from a constant. + EXPECT_TRUE(low); + EXPECT_TRUE(high); + constexpr uint64_t largest = std::numeric_limits::max(); + EXPECT_GE(Retry::drawSpacing(largest), largest - largest / 5) << "the top of the range must not wrap"; +} + TEST(CASRequests, CreateThenReplaceThenRemove) { FakeClock clock; @@ -723,6 +776,110 @@ TEST(CASRequests, AmbiguousCreateThatNeverLandedIsReissued) EXPECT_EQ(clock.sleeps.size(), 1u); } +namespace +{ + +/// Each guarded write takes `write_ms` on the injected clock and, while `failing` is positive, throws a +/// transport timeout after it. +class SlowFailingWrites final : public CountingBackend +{ +public: + explicit SlowFailingWrites(FakeClock & clock_) : clock(clock_) {} + + std::expected write(const String & key, const String & bytes, + const std::optional & expected_value, TransportAccess & access) override + { + if (expected_value) + { + clock.now += write_ms; + if (failing > 0) + { + --failing; + throw Poco::TimeoutException("injected write timeout"); + } + } + return CountingBackend::write(key, bytes, expected_value, access); + } + + uint64_t write_ms = 0; + int failing = 0; + +private: + FakeClock & clock; +}; + +} + +/// Unclear writes are reissued past the standard 90 s window, each a spacing draw after the previous +/// attempt started, and the read that settles each one is sent at once. +TEST(CASRequestsSpacing, UnclearWritesAreReissuedPastAnyWindowOneSpacingApart) +{ + FakeClock clock; + auto backend = std::make_shared(clock); + auto requests = makeRequests(backend, clock); + auto op = requests.admit(); + const Etag seen = *orThrow(op.create("k", "v1", Retry::standard()), "seed"); + constexpr int failures = 120; + backend->write_ms = 300; + backend->failing = failures; + const uint64_t gets_before = backend->getTotal(); + const uint64_t start = clock.now; + + const WriteResult result = op.replace("k", "v2", seen, Retry::untilDefinitive()); + + const auto * committed = std::get_if(&result); + ASSERT_NE(committed, nullptr); + EXPECT_EQ(committed->attempts_sent, failures + 1u); + EXPECT_GT(clock.now - start, 90'000u); + ASSERT_EQ(clock.sleeps.size(), static_cast(failures)); + for (const uint64_t pause : clock.sleeps) + { + EXPECT_GE(pause, 500u); + EXPECT_LE(pause, 900u); + } + EXPECT_EQ(backend->getTotal() - gets_before, static_cast(failures)) << "one settling read per unclear attempt"; +} + +TEST(CASRequestsSpacing, ReadRetriesAreSpaced) +{ + FakeClock clock; + auto backend = std::make_shared(); + auto requests = makeRequests(backend, clock); + auto op = requests.admit(); + (void)orThrow(op.create("k", "v", Retry::standard()), "seed"); + backend->failNextReadWith("k", std::make_exception_ptr(Poco::TimeoutException("injected read failure"))); + backend->failNextReadWith("k", std::make_exception_ptr(Poco::TimeoutException("injected read failure"))); + + const auto object = op.read("k", Retry::untilDefinitive()); + + ASSERT_TRUE(object.has_value()); + ASSERT_EQ(clock.sleeps.size(), 2u); + for (const uint64_t pause : clock.sleeps) + { + EXPECT_GE(pause, 800u); + EXPECT_LE(pause, 1'200u); + } +} + +/// Golden: a policy without spacing keeps the growing backoff. +TEST(CASRequestsSpacing, AStandardPolicyKeepsItsGrowingBackoff) +{ + FakeClock clock; + auto backend = std::make_shared(clock); + auto requests = makeRequests(backend, clock); + auto op = requests.admit(); + const Etag seen = *orThrow(op.create("k", "v1", Retry::standard()), "seed"); + backend->failing = 3; + + const WriteResult result = op.replace("k", "v2", seen, Retry::standard()); + + ASSERT_TRUE(std::holds_alternative(result)); + ASSERT_EQ(clock.sleeps.size(), 3u); + EXPECT_LE(clock.sleeps[0], 200u); + EXPECT_LE(clock.sleeps[1], 400u); + EXPECT_LE(clock.sleeps[2], 800u); +} + /// The engine's own attempt number reaches the transport through `TransportAccess::attemptNo()`, for /// every primitive -- write, read (the resolve read is its own call, with its own attempt count) and /// list. diff --git a/src/Disks/tests/gtest_cas_s3_single_attempt_client.cpp b/src/Disks/tests/gtest_cas_s3_single_attempt_client.cpp index 379b58aea3401..b19e25ed41881 100644 --- a/src/Disks/tests/gtest_cas_s3_single_attempt_client.cpp +++ b/src/Disks/tests/gtest_cas_s3_single_attempt_client.cpp @@ -53,6 +53,8 @@ #include #include #include +#include +#include /// The single-attempt client clone must cap its connect timeout at the value the mount froze at open, /// never at the disk's (possibly wider, possibly reloaded, possibly unbounded) own connect timeout. @@ -60,6 +62,7 @@ namespace ProfileEvents { extern const Event S3PutObject; +extern const Event CASConditionalWriteUnresolved; } namespace @@ -867,4 +870,33 @@ TEST(CASBackend, ConditionalPutBufferStartsAtTheBodySize) EXPECT_EQ(storage->opened_buffer_sizes, (std::vector{body.size(), 1})); } +/// `NoSuchUpload` from a conditional multipart completion does not prove that this upload created the +/// object at the key, so the write stays unresolved and the engine settles it by reading the key, which +/// here holds another writer's bytes. +TEST(CASBackend, ConditionalMultipartNoSuchUploadIsUnresolved) +{ + (void)contextForTest(); + CountingS3::ObjectStore store([](const CountingS3::Request & request, Poco::Net::HTTPServerResponse & response) + { + if (request.verb != CountingS3::Verb::CompleteMultipartUpload) + return false; + CountingS3::sendError(response, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, "NoSuchUpload"); + return true; + }); + store.put("/test-bucket/key", "another writer's bytes"); + CountingS3::CountingS3Server server(store.responder()); + auto backend = conditionalBackendForTest( + S3ObjectCreationTestStorage::makeRetryingStorage(server.getUrl(), S3ObjectCreationTestStorage::forceOnePartMultipart)); + DB::Cas::CasRequests requests(DB::Cas::BackendPtr(backend), DB::Cas::Fence::open()); + const uint64_t unresolved_before = ProfileEvents::global_counters[ProfileEvents::CASConditionalWriteUnresolved]; + + auto op = requests.admit(); + const auto result = op.create("key", std::string(100, 'c'), DB::Cas::Retry::once()); + + EXPECT_TRUE(std::holds_alternative(result)) + << "variant " << result.index() << "; another writer's object must not be taken for this upload"; + EXPECT_EQ(server.count(CountingS3::Verb::CompleteMultipartUpload), 1u); + EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASConditionalWriteUnresolved] - unresolved_before, 1u); +} + #endif diff --git a/src/Disks/tests/gtest_cas_s3_staging.cpp b/src/Disks/tests/gtest_cas_s3_staging.cpp index 74a82df51569e..430ad55262b4d 100644 --- a/src/Disks/tests/gtest_cas_s3_staging.cpp +++ b/src/Disks/tests/gtest_cas_s3_staging.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,7 @@ namespace DB::ErrorCodes { extern const int CORRUPTED_DATA; extern const int FILE_DOESNT_EXIST; + extern const int NETWORK_ERROR; } namespace @@ -100,6 +102,15 @@ class FakeNativeCopyObjectStorage final : public DB::LocalObjectStorage || (mode == DB::ObjectStorageCopyMode::NativeOnly && native_only_copy_supported); } + void removeObjectIfExists(const DB::StoredObject & object) override + { + if (throw_on_remove.load()) + throw DB::Exception(DB::ErrorCodes::NETWORK_ERROR, "injected delete failure for {}", object.remote_path); + DB::LocalObjectStorage::removeObjectIfExists(object); + } + + std::atomic throw_on_remove{false}; + private: const bool native_only_copy_supported; }; @@ -842,6 +853,29 @@ void writeThroughS3Transaction(DB::ContentAddressedTransaction & tx, const std:: buf->finalize(); } +/// Stages one blob through a real transaction, lets the transaction abort, and re-creates the exact key +/// the writer minted: the abort delete is best-effort, so a lost delete leaves the object at precisely +/// this key. Returns the key as the object storage sees it. +std::string leakWriterStagedObject(DB::IObjectStorage & object_storage, DB::ContentAddressedMetadataStorage & metadata_storage) +{ + std::string key; + { + auto tx = metadata_storage.createTransaction(); + auto & ca_tx = dynamic_cast(*tx); + writeThroughS3Transaction( + ca_tx, "a11/a11a11a1-1111-4111-8111-111111111111/all_1_1_0/data.bin", std::string(300, 'x')); + DB::RelativePathsWithMetadata staged; + object_storage.listObjects(metadata_storage.stagingKeyPrefix(), staged, /*max_keys=*/0); + if (staged.size() != 1) + throw std::runtime_error("expected exactly one staged object, got " + std::to_string(staged.size())); + key = staged.front()->relative_path; + } + auto buf = object_storage.writeObject(DB::StoredObject(key), DB::WriteMode::Rewrite); + buf->write("x", 1); + buf->finalize(); + return key; +} + } TEST(CASS3Staging, WritableS3StagingRequiresNativeOnlyCopy) @@ -933,6 +967,47 @@ TEST(CASS3Staging, ReadYourWritesReturnsStagedBytesFromS3StagingObject) EXPECT_EQ(got, payload); } +/// An aborted transaction deletes its own S3 staging object instead of leaving it for the next mount. +TEST(CASS3Staging, AbortDeletesS3Staging) +{ + auto object_storage = makeFakeNativeCopyStorage(/*native_only_copy_supported=*/true); + auto metadata_storage = makeS3StagingMetadataStorageForTest(object_storage, "mountAbort"); + metadata_storage->startup(); + + auto tx = metadata_storage->createTransaction(); + auto & ca_tx = dynamic_cast(*tx); + writeThroughS3Transaction( + ca_tx, "a11/a11a11a1-1111-4111-8111-111111111111/all_1_1_0/data.bin", std::string(300, 'x')); + + DB::RelativePathsWithMetadata staged; + object_storage->listObjects(metadata_storage->stagingKeyPrefix(), staged, /*max_keys=*/0); + ASSERT_EQ(staged.size(), 1u); + + tx.reset(); /// never committed + + DB::RelativePathsWithMetadata after; + object_storage->listObjects(metadata_storage->stagingKeyPrefix(), after, /*max_keys=*/0); + EXPECT_TRUE(after.empty()) << "an aborted transaction must not leave its S3 staging object behind"; +} + +/// Guard: the abort delete runs in a `noexcept` destructor, also after the storage was shut down with +/// the transaction still open (server shutdown). A failing delete must not escape it. +TEST(CASS3Staging, AbortAfterStorageShutdownDoesNotThrow) +{ + auto object_storage = makeFakeNativeCopyStorage(/*native_only_copy_supported=*/true); + auto metadata_storage = makeS3StagingMetadataStorageForTest(object_storage, "mountAbortLate"); + metadata_storage->startup(); + + auto tx = metadata_storage->createTransaction(); + auto & ca_tx = dynamic_cast(*tx); + writeThroughS3Transaction( + ca_tx, "a11/a11a11a1-1111-4111-8111-111111111111/all_1_1_0/data.bin", std::string(300, 'x')); + + metadata_storage->shutdown(); + object_storage->throw_on_remove = true; + EXPECT_NO_THROW(tx.reset()); +} + /// (c) `sweepOwnMountStaging` removes only objects under the given mount prefix and leaves a DIFFERENT /// mount's staging objects untouched (the lease-fence — `CASStagingSweeper.h`). TEST(CASStagingSweeper, RemovesOnlyObjectsUnderGivenMountPrefix) @@ -951,13 +1026,118 @@ TEST(CASStagingSweeper, RemovesOnlyObjectsUnderGivenMountPrefix) put(root + "/p/staging/mountA/two.tmp", "a2"); put(root + "/p/staging/mountB/three.tmp", "b1"); /// a DIFFERENT mount's staging — must survive - DB::Cas::sweepOwnMountStaging(*storage, root + "/p/staging/mountA/"); + DB::Cas::sweepOwnMountStaging(*storage, root + "/p/staging/mountA/", /*live_writer_epoch=*/1); EXPECT_FALSE(storage->exists(DB::StoredObject(root + "/p/staging/mountA/one.tmp"))); EXPECT_FALSE(storage->exists(DB::StoredObject(root + "/p/staging/mountA/two.tmp"))); EXPECT_TRUE(storage->exists(DB::StoredObject(root + "/p/staging/mountB/three.tmp"))); } +TEST(CASStagingSweeper, OldEpochSweepKeepsSuccessorStaging) +{ + auto storage = DB::Cas::tests::makeLocalObjectStorageForTest(); + const std::string root = storage->getCommonKeyPrefix(); + auto put = [&](const std::string & key) + { + auto buf = storage->writeObject(DB::StoredObject(key), DB::WriteMode::Rewrite); + buf->write("x", 1); + buf->finalize(); + }; + put(root + "/p/staging/m/1/a.tmp"); + put(root + "/p/staging/m/2/b.tmp"); + put(root + "/p/staging/m/3/c.tmp"); + + DB::Cas::sweepOwnMountStaging(*storage, root + "/p/staging/m/", /*live_writer_epoch=*/3); + + EXPECT_FALSE(storage->exists(DB::StoredObject(root + "/p/staging/m/1/a.tmp"))); + EXPECT_FALSE(storage->exists(DB::StoredObject(root + "/p/staging/m/2/b.tmp"))); + EXPECT_TRUE(storage->exists(DB::StoredObject(root + "/p/staging/m/3/c.tmp"))) << "the live epoch's own staging"; +} + +TEST(CASStagingSweeper, FlatLegacyKeysAreSwept) +{ + auto storage = DB::Cas::tests::makeLocalObjectStorageForTest(); + const std::string root = storage->getCommonKeyPrefix(); + auto buf = storage->writeObject(DB::StoredObject(root + "/p/staging/m/legacy.tmp"), DB::WriteMode::Rewrite); + buf->write("x", 1); + buf->finalize(); + DB::Cas::sweepOwnMountStaging(*storage, root + "/p/staging/m/", /*live_writer_epoch=*/1); + EXPECT_FALSE(storage->exists(DB::StoredObject(root + "/p/staging/m/legacy.tmp"))) << "a flat key counts as epoch 0"; +} + +TEST(CASStagingSweeper, NestedOtherMemberKeysAreKept) +{ + auto storage = DB::Cas::tests::makeLocalObjectStorageForTest(); + const std::string root = storage->getCommonKeyPrefix(); + auto put = [&](const std::string & key) + { + auto buf = storage->writeObject(DB::StoredObject(key), DB::WriteMode::Rewrite); + buf->write("x", 1); + buf->finalize(); + }; + put(root + "/p/staging/a/b/1/x.tmp"); /// member `a/b`, nested under `a` + put(root + "/p/staging/a/1/own.tmp"); + DB::Cas::sweepOwnMountStaging(*storage, root + "/p/staging/a/", /*live_writer_epoch=*/5); + EXPECT_TRUE(storage->exists(DB::StoredObject(root + "/p/staging/a/b/1/x.tmp"))) << "`b` is not an epoch"; + EXPECT_FALSE(storage->exists(DB::StoredObject(root + "/p/staging/a/1/own.tmp"))); +} + +TEST(CASStagingSweeper, DeeperPathsUnderAnEpochDirectoryAreKept) +{ + auto storage = DB::Cas::tests::makeLocalObjectStorageForTest(); + const std::string root = storage->getCommonKeyPrefix(); + auto buf = storage->writeObject(DB::StoredObject(root + "/p/staging/a/7/x/y.tmp"), DB::WriteMode::Rewrite); + buf->write("x", 1); + buf->finalize(); + DB::Cas::sweepOwnMountStaging(*storage, root + "/p/staging/a/", /*live_writer_epoch=*/9); + EXPECT_TRUE(storage->exists(DB::StoredObject(root + "/p/staging/a/7/x/y.tmp"))) << "member `a/7`'s key, not `a`'s"; +} + +/// The staging key's epoch and the admission generation the finalize rechecks come from one capture: a +/// self-remount between the two reads refuses the write instead of staging new-generation bytes under the +/// old epoch. Without the hook, the key carries the live epoch. +TEST(ContentAddressedTransaction, StagingEpochMatchesAdmissionGeneration) +{ + auto object_storage = makeFakeNativeCopyStorage(/*native_only_copy_supported=*/true); + auto metadata_storage = makeS3StagingMetadataStorageForTest(object_storage, "mountE"); + metadata_storage->startup(); + const DB::Cas::PoolPtr pool = metadata_storage->poolForTest(); + const uint64_t epoch_before = pool->liveWriterEpoch(); + const std::string path = "a11/a11a11a1-1111-4111-8111-111111111111/all_1_1_0/data.bin"; + + /// Kept alive to the end: an abandoned transaction deletes its staging object, and the final + /// assertion counts this write's key under the old epoch. + auto first_tx = metadata_storage->createTransaction(); + { + auto & ca_tx = dynamic_cast(*first_tx); + writeThroughS3Transaction(ca_tx, path, std::string(300, 'x')); + DB::RelativePathsWithMetadata staged; + object_storage->listObjects(metadata_storage->stagingKeyPrefix(), staged, /*max_keys=*/0); + ASSERT_EQ(staged.size(), 1u); + EXPECT_NE(staged.front()->relative_path.find("/" + std::to_string(epoch_before) + "/"), std::string::npos) + << staged.front()->relative_path; + } + + const String mount_key = pool->layout().mountKey("mountE"); + bool remounted = false; + pool->setIncarnationCaptureHookForTest([&] + { + if (std::exchange(remounted, true)) + return; + DB::Cas::tests::fenceOutMountForTest(*pool->poolBackendPtr(), mount_key); + try { pool->renewWatermarkOnce(); } catch (const DB::Exception &) {} + ASSERT_TRUE(pool->tryRemountOnce()); + }); + auto tx = metadata_storage->createTransaction(); + auto & ca_tx = dynamic_cast(*tx); + EXPECT_THROW(ca_tx.writeFile(path, 65536, DB::WriteMode::Rewrite, {}), DB::Exception); + ASSERT_TRUE(remounted); + EXPECT_EQ(pool->liveWriterEpoch(), epoch_before + 1); + DB::RelativePathsWithMetadata staged; + object_storage->listObjects(metadata_storage->stagingKeyPrefix() + std::to_string(epoch_before) + "/", staged, 0); + EXPECT_EQ(staged.size(), 1u) << "only the first write's key; the refused write staged nothing under the old epoch"; +} + /// (d) GC's blob-discovery LISTs ONLY `Layout::blobsPrefix()` (`/blobs/`) — a top-level prefix /// strictly disjoint from the S3-staging area (`/staging//`), so a staging object can /// never be listed, HEAD'd, or condemned as an orphan blob by GC's fold (`CasGc.cpp`, `CasFsck.cpp`). @@ -983,7 +1163,6 @@ TEST(CASS3Staging, GcBlobDiscoveryPrefixExcludesStagingObjects) std::string staging_prefix = full_staging_prefix.substr(physical_root.size()); if (!staging_prefix.empty() && staging_prefix.front() == '/') staging_prefix.erase(0, 1); - staging_prefix += "/"; const std::string staging_key = staging_prefix + "aaa.tmp"; const DB::Cas::Layout layout(metadata_storage->poolForTest()->poolConfig().pool_prefix); @@ -996,6 +1175,46 @@ TEST(CASS3Staging, GcBlobDiscoveryPrefixExcludesStagingObjects) EXPECT_FALSE(staging_key.starts_with(blobs_prefix)); } +/// Guard: the drain lists `/staging//` as a LOGICAL key and the backend applies the +/// local root itself, while the writer mints its keys through `physicalKey`. Local prefix: +/// `EmulatedSingleProcess` over a `LocalObjectStorage`, whose physical keys carry its root. +TEST(CASDecommissionStaging, StagingDrainRemovesWhatTheWriterStagedUnderTheLocalPrefix) +{ + auto object_storage = makeFakeNativeCopyStorage(/*native_only_copy_supported=*/true); + auto metadata_storage = makeS3StagingMetadataStorageForTest(object_storage, "victimLocal"); + metadata_storage->startup(); + const std::string leaked = leakWriterStagedObject(*object_storage, *metadata_storage); + metadata_storage->shutdown(); /// clean farewell: the member is dead and claimable without a wait + ASSERT_TRUE(object_storage->exists(DB::StoredObject(leaked))); + + auto backend = std::make_shared( + object_storage, DB::Cas::ObjectStorageBackend::Mode::EmulatedSingleProcess); + const DB::Cas::DecommissionReport report = DB::Cas::decommissionPoolMember( + backend, DB::Cas::PoolConfig{.pool_prefix = "pool", .server_root_id = "admin"}, "victimLocal"); + + EXPECT_EQ(report.staging_objects_removed, 1u) << "the drain did not see the writer's key space"; + EXPECT_FALSE(object_storage->exists(DB::StoredObject(leaked))); +} + +/// Guard: a staging object whose abort delete was lost is reclaimed by the next mount of the same +/// `server_root_id`, through the same prefix the writer minted it under. +TEST(CASS3Staging, MountStartSweepsALostAbortDelete) +{ + auto object_storage = makeFakeNativeCopyStorage(/*native_only_copy_supported=*/true); + std::string leaked; + { + auto first_mount = makeS3StagingMetadataStorageForTest(object_storage, "mountSweep"); + first_mount->startup(); + leaked = leakWriterStagedObject(*object_storage, *first_mount); + first_mount->shutdown(); + } + ASSERT_TRUE(object_storage->exists(DB::StoredObject(leaked))); + + auto second_mount = makeS3StagingMetadataStorageForTest(object_storage, "mountSweep"); + second_mount->startup(); + EXPECT_FALSE(object_storage->exists(DB::StoredObject(leaked))); +} + #if USE_AWS_S3 namespace @@ -1335,4 +1554,24 @@ TEST(CASS3Staging, GenerationBackendMayUseNativeOnlyCopy) EXPECT_EQ(staged.size(), 1u); } +/// Native prefix: keys pass through unchanged on both sides, so a `physicalKey` that started adding a +/// root for Native, or a drain that started adding one, would miss the writer's objects. +TEST(CASDecommissionStaging, StagingDrainRemovesWhatTheWriterStagedUnderTheNativePrefix) +{ + auto object_storage = makeFakeGenerationObjectStorageForTest(); + auto metadata_storage = makeS3StagingMetadataStorageForTest(object_storage, "victimNative"); + metadata_storage->startup(); + const std::string leaked = leakWriterStagedObject(*object_storage, *metadata_storage); + metadata_storage->shutdown(); + ASSERT_TRUE(object_storage->exists(DB::StoredObject(leaked))); + + auto backend = std::make_shared( + object_storage, DB::Cas::ObjectStorageBackend::Mode::Native); + const DB::Cas::DecommissionReport report = DB::Cas::decommissionPoolMember( + backend, DB::Cas::PoolConfig{.pool_prefix = "pool", .server_root_id = "admin"}, "victimNative"); + + EXPECT_EQ(report.staging_objects_removed, 1u) << "the drain did not see the writer's key space"; + EXPECT_FALSE(object_storage->exists(DB::StoredObject(leaked))); +} + #endif diff --git a/src/Disks/tests/gtest_cas_server_root_format.cpp b/src/Disks/tests/gtest_cas_server_root_format.cpp index bf04753c9b9f1..691174ad20ad0 100644 --- a/src/Disks/tests/gtest_cas_server_root_format.cpp +++ b/src/Disks/tests/gtest_cas_server_root_format.cpp @@ -1,8 +1,11 @@ #include "cas_format_test_battery.h" #include #include +#include #include +#include #include +#include using namespace DB::Cas; @@ -46,10 +49,62 @@ TEST(CASFormatBattery, ServerEpoch) { ServerEpoch e; e.next_writer_epoch = 7; + e.allocation_nonce = hexToU128("0123456789abcdeffedcba9876543210"); + const String golden = currentFormatHeader("cas_epoch") + + "{\"next_writer_epoch\":\"7\",\"allocation_nonce\":\"0123456789abcdeffedcba9876543210\"}\n"; + const ServerEpoch back = decodeServerEpoch(golden); + EXPECT_EQ(back.next_writer_epoch, 7u); + EXPECT_EQ(back.allocation_nonce, e.allocation_nonce); runFormatBattery({FormatId::ServerEpoch, [&] { return sealObject(FormatId::ServerEpoch, encodeServerEpoch(e)); }, [](std::string_view s) { decodeServerEpoch(std::string(openObject(FormatId::ServerEpoch, s))); }, - currentFormatHeader("cas_epoch") + "{\"next_writer_epoch\":\"7\"}\n"}); + golden}); +} + +/// The counter body written by 26.6.4, before the allocation nonce existed. +TEST(CASServerEpochFormat, BodyWithoutNonceDecodes) +{ + const ServerEpoch e = decodeServerEpoch(currentFormatHeader("cas_epoch") + "{\"next_writer_epoch\":\"7\"}\n"); + EXPECT_EQ(e.next_writer_epoch, 7u); + EXPECT_FALSE(e.allocation_nonce.has_value()); +} + +/// The 26.6.4 decoder's loop: it knows only `next_writer_epoch` and hands every other key to `skipUnknown`. +TEST(CASServerEpochFormat, NonceBodyDecodesWithThe2664KeySet) +{ + const String body = encodeServerEpoch(ServerEpoch{ + .next_writer_epoch = 7, .allocation_nonce = hexToU128("0123456789abcdeffedcba9876543210")}); + DB::ReadBufferFromMemory in(body.data(), body.size()); + expectHeaderLine(in, FormatId::ServerEpoch); + const String line = readLine(in, traitsFor(FormatId::ServerEpoch).line_cap, "server-epoch"); + DB::ReadBufferFromMemory line_in(line.data(), line.size()); + JsonObjectReader r(line_in, "server-epoch"); + std::optional next; + String key; + while (r.nextKey(key)) + { + if (key == "next_writer_epoch") + next = r.readU64String(); + else + r.skipUnknown(key); + } + EXPECT_EQ(next, 7u); + EXPECT_TRUE(line_in.eof()); + EXPECT_TRUE(in.eof()); +} + +TEST(CASServerEpochFormat, ZeroNextWriterEpochIsRejected) +{ + const String data = currentFormatHeader("cas_epoch") + "{\"next_writer_epoch\":\"0\"}\n"; + try + { + decodeServerEpoch(data); + FAIL() << "expected CORRUPTED_DATA"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::CORRUPTED_DATA); + } } CAS_BATTERY_COVERS(MountLease); @@ -167,3 +222,61 @@ TEST(CASMountLeaseFormat, RejectsMissingIdentityFields) expectMessage(header + "{" + uuid + attempt + rest + "\n", "CAS mount-lease: missing writer_epoch"); expectMessage(header + "{" + uuid + epoch + rest + "\n", "CAS mount-lease: missing or zero write_attempt_id"); } + +TEST(CASOwnerFormat, LifeAndVerifiedRoundTrip) +{ + OwnerObject o; + o.server_uuid = hexToU128("0123456789abcdeffedcba9876543210"); + o.retired_at_ms = 1752537600000ULL; + o.owner_life = 7; + o.retirement_verified = true; + EXPECT_EQ(encodeOwner(o), currentFormatHeader("cas_owner") + + "{\"server_uuid\":\"0123456789abcdeffedcba9876543210\",\"retired_at_ms\":1752537600000," + "\"owner_life\":\"7\",\"retirement_verified\":true}\n"); + const OwnerObject back = decodeOwner(encodeOwner(o)); + EXPECT_EQ(back.owner_life, 7u); + EXPECT_TRUE(back.retirement_verified); + EXPECT_EQ(back.retired_at_ms, o.retired_at_ms); +} + +/// The owner body written by 26.6.4 and by a never-retired owner: no life, no mark. +TEST(CASOwnerFormat, BodyWithoutLifeReadsAsZeroAndUnverified) +{ + const OwnerObject o = decodeOwner(currentFormatHeader("cas_owner") + + "{\"server_uuid\":\"0123456789abcdeffedcba9876543210\"}\n"); + EXPECT_EQ(o.owner_life, 0u); + EXPECT_FALSE(o.retirement_verified); + OwnerObject zero; + zero.server_uuid = o.server_uuid; + EXPECT_EQ(encodeOwner(zero), currentFormatHeader("cas_owner") + + "{\"server_uuid\":\"0123456789abcdeffedcba9876543210\"}\n") + << "a zero life and a false mark are not written, so the 26.6.4 bytes stay reachable"; +} + +/// The 26.6.4 decoder's loop knows `server_uuid` and `retired_at_ms` and hands every other key to `skipUnknown`. +TEST(CASOwnerFormat, NewFieldsDecodeWithThe2664KeySet) +{ + const String body = encodeOwner(OwnerObject{ + .server_uuid = hexToU128("0123456789abcdeffedcba9876543210"), + .retired_at_ms = 5, .owner_life = 3, .retirement_verified = true}); + DB::ReadBufferFromMemory in(body.data(), body.size()); + expectHeaderLine(in, FormatId::Owner); + const String line = readLine(in, traitsFor(FormatId::Owner).line_cap, "owner"); + DB::ReadBufferFromMemory line_in(line.data(), line.size()); + JsonObjectReader r(line_in, "owner"); + String key; + std::optional uuid; + std::optional retired; + while (r.nextKey(key)) + { + if (key == "server_uuid") + uuid = r.readHex128(); + else if (key == "retired_at_ms") + retired = r.readU64Number(); + else + r.skipUnknown(key); + } + EXPECT_EQ(uuid, hexToU128("0123456789abcdeffedcba9876543210")); + EXPECT_EQ(retired, 5u); + EXPECT_TRUE(line_in.eof()); +} diff --git a/src/Disks/tests/gtest_cas_settings.cpp b/src/Disks/tests/gtest_cas_settings.cpp index 61c800b7491ee..0a15ba363667b 100644 --- a/src/Disks/tests/gtest_cas_settings.cpp +++ b/src/Disks/tests/gtest_cas_settings.cpp @@ -26,6 +26,7 @@ namespace DB::ContentAddressedSetting extern const ContentAddressedSettingsUInt64 gc_shards; extern const ContentAddressedSettingsUInt64 gc_interval_sec; extern const ContentAddressedSettingsUInt64 gc_bulk_delete_chunk_keys; + extern const ContentAddressedSettingsUInt64 mount_lease_ttl_ms; extern const ContentAddressedSettingsString scratch_path; extern const ContentAddressedSettingsBool unsafe_remount_no_delay; extern const ContentAddressedSettingsBool allow_packed_parts; @@ -627,3 +628,24 @@ TEST(CASContentAddressedSettings, UnsafeRemountNoDelayIsOffByDefault) EXPECT_TRUE(s[ContentAddressedSetting::unsafe_remount_no_delay].value); } } + +TEST(CASContentAddressedSettings, MountLeaseTtlAboveTheMaximumIsClampedWithOneWarning) +{ + ScopedCasSettingsLogCapture capture; + auto cfg = makeConfig("srv13600000"); + ContentAddressedSettings s; + s.loadFromConfig(*cfg, "disk", "/data", "/data/scratch", identity_macros); + EXPECT_EQ(s[ContentAddressedSetting::mount_lease_ttl_ms].value, 600'000u); + const auto log = capture.captured(); + EXPECT_EQ(countOccurrences(log, "cas_mount_lease_ttl_ms=3600000 exceeds the maximum of 600000 ms; using 600000 ms"), 1u) << log; +} + +TEST(CASContentAddressedSettings, MountLeaseTtlAtTheMaximumIsKeptSilently) +{ + ScopedCasSettingsLogCapture capture; + auto cfg = makeConfig("srv1600000"); + ContentAddressedSettings s; + s.loadFromConfig(*cfg, "disk", "/data", "/data/scratch", identity_macros); + EXPECT_EQ(s[ContentAddressedSetting::mount_lease_ttl_ms].value, 600'000u); + EXPECT_EQ(countOccurrences(capture.captured(), "exceeds the maximum"), 0u) << capture.captured(); +} diff --git a/src/Disks/tests/gtest_cas_startup_stale_mounts.cpp b/src/Disks/tests/gtest_cas_startup_stale_mounts.cpp new file mode 100644 index 0000000000000..bc296e3d58c8d --- /dev/null +++ b/src/Disks/tests/gtest_cas_startup_stale_mounts.cpp @@ -0,0 +1,701 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// Server start re-creates the `disk = disk(...)` custom disks of every table it loads. A CAS disk +/// whose previous process died without a farewell waits out the stale mount lease inside that +/// creation. These tests pin how such waits compose with the rest of startup. + +namespace DB::ErrorCodes +{ +extern const int BAD_ARGUMENTS; +extern const int UNKNOWN_DISK; +} + +namespace +{ + +namespace fs = std::filesystem; +using namespace DB::Cas; + +constexpr uint64_t lease_ttl_ms = 3000; +constexpr uint64_t renew_period_ms = 600; +/// `mountObservationThresholdMs`: TTL + TTL/20 + renew_period/2. +constexpr uint64_t observation_ms = lease_ttl_ms + lease_ttl_ms / 20 + renew_period_ms / 2; + +uint64_t elapsedMs(std::chrono::steady_clock::time_point since) +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - since).count()); +} + +uint64_t wallNowMs() +{ + return static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); +} + +struct StaleCasDisk +{ + String name; + String server_root_id; + fs::path pool_dir; + fs::path scratch_dir; + + std::map settings() const + { + return { + {"type", "object_storage"}, + {"object_storage_type", "local"}, + {"metadata_type", "cas"}, + {"path", pool_dir.string() + "/"}, + {"cas_server_root_id", server_root_id}, + {"cas_scratch_path", scratch_dir.string() + "/"}, + {"cas_mount_lease_ttl_ms", std::to_string(lease_ttl_ms)}, + {"cas_mount_renew_period_ms", std::to_string(renew_period_ms)}, + {"cas_attempt_timeout_ms", "200"}, + {"cas_lease_safety_margin_ms", "200"}, + {"cas_gc_enabled", "0"}, + }; + } + + /// The table-metadata form: what server start parses from `SETTINGS disk = disk(...)`. + DB::ASTPtr diskFunctionAst() const + { + String text = "disk(name = '" + name + "'"; + for (const auto & [key, value] : settings()) + text += ", " + key + " = '" + value + "'"; + text += ")"; + DB::ParserFunction parser; + return DB::parseQuery(parser, text, "", 0, 0, 0); + } + + Poco::AutoPtr xmlConfig() const + { + std::ostringstream xml; + xml << ""; + for (const auto & [key, value] : settings()) + xml << "<" << key << ">" << value << ""; + xml << ""; + std::istringstream in(xml.str()); + return Poco::AutoPtr(new Poco::Util::XMLConfiguration(in)); + } +}; + +/// Leaves `disk` as a process killed after it minted a writer epoch and claimed the mount: a live- +/// looking lease with no farewell, held by this server's own uuid under an older epoch. +void seedStaleMount(const StaleCasDisk & disk) +{ + { + auto config = disk.xmlConfig(); + auto seeded = DB::DiskFactory::instance().create( + disk.name + "_seed", *config, "disk", getContext().context, {}, /*attach=*/true, /*custom_disk=*/true); + seeded->shutdown(); + } + + auto object_storage = std::make_shared( + DB::LocalObjectStorageSettings(disk.name + "_seed_backend", disk.pool_dir.string() + "/", /*read_only_=*/false)); + auto backend = std::make_shared(object_storage, ObjectStorageBackend::Mode::EmulatedSingleProcess); + DB::Cas::tests::OperationForTest op(backend); + const Layout layout{"ca"}; + + const auto mount = (*op).read(layout.mountKey(disk.server_root_id), Retry::standard()); + ASSERT_TRUE(mount.has_value()); + const MountLease farewell = decodeMountLease(mount->bytes); + ASSERT_EQ(farewell.min_active_build_sequence, std::numeric_limits::max()) + << "the seeding open must end in a clean farewell"; + + const auto epoch_object = (*op).read(layout.epochKey(disk.server_root_id), Retry::standard()); + ASSERT_TRUE(epoch_object.has_value()); + const uint64_t crashed_epoch = decodeServerEpoch(epoch_object->bytes).next_writer_epoch; + ASSERT_TRUE(std::holds_alternative((*op).replace( + layout.epochKey(disk.server_root_id), encodeServerEpoch(ServerEpoch{.next_writer_epoch = crashed_epoch + 1}), + epoch_object->etag, Retry::standard()))); + + const MountClaimResult claim = claimMount( + *op, layout, disk.server_root_id, farewell.server_uuid, crashed_epoch, wallNowMs(), lease_ttl_ms); + ASSERT_EQ(claim.kind, MountClaimResult::Claimed); +} + +/// Signals when a CAS mount starts waiting out a stale lease. +class StaleMountWaitChannel : public Poco::Channel +{ +public: + void log(const Poco::Message & msg) override + { + if (msg.getText().find("stale-looking mount lease") == std::string::npos) + return; + std::lock_guard lock(mutex); + ++waits_started; + cv.notify_all(); + } + + bool waitForWaitStarted(size_t count) + { + std::unique_lock lock(mutex); + return cv.wait_for(lock, std::chrono::seconds(30), [&] { return waits_started >= count; }); + } + +private: + std::mutex mutex; + std::condition_variable cv; + size_t waits_started = 0; +}; + +class CASStartupStaleMounts : public ::testing::Test +{ +public: + static void SetUpTestSuite() + { + DB::ServerUUID::setRandomForUnitTests(); + DB::clearDiskRegistry(); + DB::registerDisks(/*global_skip_access_check=*/true); + } + + static void TearDownTestSuite() + { + DB::clearDiskRegistry(); + } + +protected: + void SetUp() override + { + root = fs::temp_directory_path() + / ("cas_startup_stale_mounts_" + std::to_string(::getpid()) + "_" + + ::testing::UnitTest::GetInstance()->current_test_info()->name()); + fs::remove_all(root); + fs::create_directories(root); + } + + void TearDown() override + { + for (const auto & disk : disks) + if (auto created = getContext().context->getDisksMap(); created.contains(disk.name)) + created.at(disk.name)->shutdown(); + /// Restored only after the disks stop, so no CAS worker logs while the channel is swapped. + if (stale_mount_waits) + { + Poco::Logger & pool_logger = Poco::Logger::get("CasPool"); + pool_logger.setChannel(previous_pool_channel); + pool_logger.setLevel(previous_pool_level); + } + fs::remove_all(root); + } + + /// Spins until `count` requests wait for the creation of `name`: no timing assumption, and a creator + /// that never gets its waiter fails the test by the deadline instead of passing. + static void waitForWaiters(const String & name, size_t count) + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (getContext().context->getDiskCreationWaitersForTest(name) < count) + { + ASSERT_LT(std::chrono::steady_clock::now(), deadline) << "no request joined the creation"; + std::this_thread::yield(); + } + } + + /// Disks stay in the global context after a test, so every name is unique within the process. + static String uniqueDiskName(const String & prefix) + { + static std::atomic counter = 0; + return prefix + "_" + ::testing::UnitTest::GetInstance()->current_test_info()->name() + "_" + std::to_string(++counter); + } + + StaleCasDisk & addStaleDisk(const String & suffix) + { + StaleCasDisk disk{ + .name = uniqueDiskName("cas_stale_" + suffix), + .server_root_id = "stale_" + suffix, + .pool_dir = root / ("pool_" + suffix), + .scratch_dir = root / ("scratch_" + suffix), + }; + fs::create_directories(disk.scratch_dir); + seedStaleMount(disk); + return disks.emplace_back(std::move(disk)); + } + + StaleMountWaitChannel & watchStaleMountWaits() + { + Poco::Logger & pool_logger = Poco::Logger::get("CasPool"); + previous_pool_channel = Poco::AutoPtr(pool_logger.getChannel(), /*shared=*/true); + previous_pool_level = pool_logger.getLevel(); + stale_mount_waits = new StaleMountWaitChannel; + pool_logger.setChannel(stale_mount_waits); + pool_logger.setLevel(Poco::Message::PRIO_INFORMATION); + return *stale_mount_waits; + } + + fs::path root; + std::vector disks; + Poco::AutoPtr stale_mount_waits; + Poco::AutoPtr previous_pool_channel; + int previous_pool_level = 0; +}; + +} + +/// Each stale mount must be waited out once, but the waits of different disks are independent. +/// Waiting them one after another makes server start N × the observation window. +TEST_F(CASStartupStaleMounts, StaleMountsOfDifferentDisksAreWaitedOutConcurrently) +{ + constexpr size_t disk_count = 3; + for (size_t i = 0; i < disk_count; ++i) + addStaleDisk(std::to_string(i)); + ASSERT_FALSE(HasFatalFailure()); + + std::vector per_disk_ms(disk_count, 0); + std::vector errors(disk_count); + const auto start = std::chrono::steady_clock::now(); + { + std::vector loaders; + for (size_t i = 0; i < disk_count; ++i) + loaders.emplace_back([&, i] + { + try + { + DB::DiskFromAST::createCustomDisk(disks[i].diskFunctionAst(), getContext().context, /*attach=*/true); + } + catch (...) + { + errors[i] = std::current_exception(); + } + per_disk_ms[i] = elapsedMs(start); + }); + for (auto & loader : loaders) + loader.join(); + } + const uint64_t total_ms = elapsedMs(start); + + for (size_t i = 0; i < disk_count; ++i) + { + ASSERT_FALSE(errors[i]) << DB::getExceptionMessage(errors[i], false); + EXPECT_GE(per_disk_ms[i], observation_ms) << "disk " << i << " had no stale mount to wait out"; + } + EXPECT_LT(total_ms, 2 * observation_ms) + << disk_count << " stale mounts took " << total_ms << " ms to open; one observation window is " + << observation_ms << " ms, so the waits ran one after another"; +} + +/// A disk lookup by an unrelated table load must not queue behind a stale-mount wait: the wait is +/// up to a full lease TTL, and every table load and system log flush resolves disks. +TEST_F(CASStartupStaleMounts, DiskLookupDoesNotWaitForAStaleMountObservation) +{ + const StaleCasDisk & stale = addStaleDisk("0"); + ASSERT_FALSE(HasFatalFailure()); + StaleMountWaitChannel & waits = watchStaleMountWaits(); + + std::exception_ptr error; + std::thread loader([&] + { + try + { + DB::DiskFromAST::createCustomDisk(stale.diskFunctionAst(), getContext().context, /*attach=*/true); + } + catch (...) + { + error = std::current_exception(); + } + }); + + const bool wait_started = waits.waitForWaitStarted(1); + uint64_t lookup_ms = 0; + if (wait_started) + { + const auto lookup_start = std::chrono::steady_clock::now(); + (void)getContext().context->getDisksMap(); + lookup_ms = elapsedMs(lookup_start); + } + loader.join(); + + ASSERT_FALSE(error) << DB::getExceptionMessage(error, false); + ASSERT_TRUE(wait_started) << "the stale mount was never observed"; + EXPECT_LT(lookup_ms, observation_ms / 3) + << "a disk lookup took " << lookup_ms << " ms while a stale mount was being observed for up to " + << observation_ms << " ms"; +} + +/// Two table loads that name the same custom disk must share one creation: a second open of the +/// same CAS root while the first is still waiting out the stale mount would be a live twin. +TEST_F(CASStartupStaleMounts, ConcurrentCreationsOfOneDiskShareOneOpen) +{ + const StaleCasDisk & stale = addStaleDisk("0"); + ASSERT_FALSE(HasFatalFailure()); + StaleMountWaitChannel & waits = watchStaleMountWaits(); + + const auto config = stale.xmlConfig(); + std::atomic creator_calls = 0; + const auto creator = [&](const DB::DisksMap & disks_map) + { + ++creator_calls; + return DB::DiskFactory::instance().create( + stale.name, *config, "disk", getContext().context, disks_map, /*attach=*/true, /*custom_disk=*/true); + }; + + DB::DiskPtr first_disk; + DB::DiskPtr second_disk; + std::exception_ptr first_error; + std::exception_ptr second_error; + std::thread first([&] + { + try + { + first_disk = getContext().context->getOrCreateDisk(stale.name, creator); + } + catch (...) + { + first_error = std::current_exception(); + } + }); + /// Without this wait the second request could find the disk already created. + const bool first_is_waiting = waits.waitForWaitStarted(1); + std::thread second([&] + { + try + { + second_disk = getContext().context->getOrCreateDisk(stale.name, creator); + } + catch (...) + { + second_error = std::current_exception(); + } + }); + first.join(); + second.join(); + + ASSERT_TRUE(first_is_waiting) << "the stale mount was never observed"; + ASSERT_FALSE(first_error) << DB::getExceptionMessage(first_error, false); + ASSERT_FALSE(second_error) << DB::getExceptionMessage(second_error, false); + EXPECT_EQ(creator_calls.load(), 1u); + EXPECT_EQ(first_disk, second_disk); + EXPECT_EQ(first_disk, getContext().context->getDisk(stale.name)); +} + +/// A failed creation is not remembered: the next request for the name creates the disk again. +TEST_F(CASStartupStaleMounts, FailedDiskCreationIsRetriedByTheNextRequest) +{ + const String name = uniqueDiskName("failed_creation"); + std::vector calls; + EXPECT_THROW( + getContext().context->getOrCreateDisk(name, [&](const DB::DisksMap &) -> DB::DiskPtr + { + calls.push_back("first"); + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "creation failed"); + }), + DB::Exception); + EXPECT_FALSE(getContext().context->getDisksMap().contains(name)); + EXPECT_THROW( + getContext().context->getOrCreateDisk(name, [&](const DB::DisksMap &) -> DB::DiskPtr + { + calls.push_back("second"); + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "creation failed again"); + }), + DB::Exception); + EXPECT_EQ(calls, (std::vector{"first", "second"})); +} + +/// A request that arrives while another creation of the same name fails gets that failure without +/// running its own creator. Each caller gets its own copy of the exception, of the original type. +TEST_F(CASStartupStaleMounts, ConcurrentRequestsShareOneFailure) +{ + const String name = uniqueDiskName("shared_failure"); + std::mutex mutex; + std::condition_variable cv; + bool first_started = false; + std::atomic creator_calls = 0; + + const auto failing_creator = [&](const DB::DisksMap &) -> DB::DiskPtr + { + ++creator_calls; + { + std::lock_guard lock(mutex); + first_started = true; + cv.notify_all(); + } + waitForWaiters(name, 1); + DB::ErrnoException::throwWithErrno(DB::ErrorCodes::BAD_ARGUMENTS, ENOSPC, "creation failed"); + }; + + struct Outcome + { + bool got_errno_exception = false; + int code = 0; + int saved_errno = 0; + String message; + }; + const auto request = [&](const String & caller, Outcome & outcome) + { + try + { + getContext().context->getOrCreateDisk(name, failing_creator); + } + catch (DB::ErrnoException & e) + { + outcome.got_errno_exception = true; + outcome.code = e.code(); + outcome.saved_errno = e.getErrno(); + e.addMessage("seen by " + caller); + outcome.message = e.message(); + } + catch (...) + { + outcome.message = DB::getCurrentExceptionMessage(false); + } + }; + + Outcome first_outcome; + Outcome second_outcome; + std::thread first([&] { request("first", first_outcome); }); + bool first_is_creating = false; + { + std::unique_lock lock(mutex); + first_is_creating = cv.wait_for(lock, std::chrono::seconds(30), [&] { return first_started; }); + } + std::thread second([&] { request("second", second_outcome); }); + first.join(); + second.join(); + + ASSERT_TRUE(first_is_creating) << "the first creator never started"; + EXPECT_EQ(creator_calls.load(), 1u); + for (const Outcome * outcome : {&first_outcome, &second_outcome}) + { + EXPECT_TRUE(outcome->got_errno_exception) << outcome->message; + EXPECT_EQ(outcome->code, DB::ErrorCodes::BAD_ARGUMENTS); + EXPECT_EQ(outcome->saved_errno, ENOSPC); + } + EXPECT_TRUE(first_outcome.message.contains("seen by first")) << first_outcome.message; + EXPECT_FALSE(first_outcome.message.contains("seen by second")) << first_outcome.message; + EXPECT_TRUE(second_outcome.message.contains("seen by second")) << second_outcome.message; + EXPECT_FALSE(second_outcome.message.contains("seen by first")) << second_outcome.message; + EXPECT_FALSE(getContext().context->getDisksMap().contains(name)); +} + +/// A configuration reload during a custom disk creation must not open a configured disk of the +/// same name: both could describe the same storage. +TEST_F(CASStartupStaleMounts, ConfigReloadSkipsADiskBeingCreated) +{ + const String name = uniqueDiskName("reload_race"); + const fs::path config_disk_path = root / "config_disk"; + const fs::path custom_disk_path = root / "custom_disk"; + fs::create_directories(config_disk_path); + fs::create_directories(custom_disk_path); + + std::istringstream xml( + "<" + name + ">local" + config_disk_path.string() + + "/"); + Poco::AutoPtr reloaded_config(new Poco::Util::XMLConfiguration(xml)); + + DB::DiskPtr created; + const auto disk = getContext().context->getOrCreateDisk(name, [&](const DB::DisksMap &) -> DB::DiskPtr + { + getContext().context->updateStorageConfiguration(*reloaded_config); + created = std::make_shared(name, custom_disk_path.string() + "/"); + return created; + }); + + EXPECT_EQ(disk, created); + EXPECT_EQ(getContext().context->getDisk(name), created); +} + +/// A reload during a creation drops only the storage policies that name the disk being created. +/// Policies on other disks are applied, and a policy that already exists keeps its previous disks. +TEST_F(CASStartupStaleMounts, ConfigReloadKeepsPoliciesThatDoNotNameTheDiskBeingCreated) +{ + const String name = uniqueDiskName("reload_policies"); + const fs::path other_path = root / "other_disk"; + const fs::path config_disk_path = root / "config_disk"; + const fs::path custom_disk_path = root / "custom_disk"; + for (const auto & path : {other_path, config_disk_path, custom_disk_path}) + fs::create_directories(path); + + const auto reload_config = [&](const String & policies, bool with_creating_disk) + { + std::ostringstream xml; + xml << "" + << "local" << other_path.string() << "/"; + if (with_creating_disk) + xml << "<" << name << ">local" << config_disk_path.string() << "/"; + xml << "" << policies << ""; + std::istringstream in(xml.str()); + return Poco::AutoPtr(new Poco::Util::XMLConfiguration(in)); + }; + const auto policy = [](const String & policy_name, const String & disks) + { + return "<" + policy_name + ">" + disks + ""; + }; + + /// Policies outlive the test in the global context and a reload cannot drop one, so names are + /// unique per run and every reload repeats the policies of earlier runs. + static String earlier_policies; + const String p_old = name + "_old"; + const String p_new_on_creating = name + "_new_on_creating"; + const String p_new_on_other = name + "_new_on_other"; + + getContext().context->getPoliciesMap(); + getContext().context->updateStorageConfiguration(*reload_config(earlier_policies + policy(p_old, "other"), false)); + ASSERT_TRUE(getContext().context->getPoliciesMap().contains(p_old)); + + getContext().context->getOrCreateDisk(name, [&](const DB::DisksMap &) -> DB::DiskPtr + { + const String policies = earlier_policies + policy(p_old, "other" + name + "") + + policy(p_new_on_creating, "" + name + "") + policy(p_new_on_other, "other"); + getContext().context->updateStorageConfiguration(*reload_config(policies, true)); + return std::make_shared(name, custom_disk_path.string() + "/"); + }); + + earlier_policies += policy(p_old, "other") + policy(p_new_on_other, "other"); + + const auto policies = getContext().context->getPoliciesMap(); + EXPECT_TRUE(policies.contains(p_new_on_other)); + EXPECT_FALSE(policies.contains(p_new_on_creating)); + ASSERT_TRUE(policies.contains(p_old)); + EXPECT_EQ(policies.at(p_old)->getDisks().size(), 1u); +} + +/// Has no `clone` or `rethrow`, so any copy of it is sliced to `DB::Exception`. +struct ExceptionWithoutRethrow : DB::Exception +{ + using DB::Exception::Exception; +}; + +/// A failed creation rethrows the creator's own exception, not a copy sliced to a base type. +TEST_F(CASStartupStaleMounts, FailedCreationKeepsTheExceptionTypeForTheCreator) +{ + EXPECT_THROW( + getContext().context->getOrCreateDisk(uniqueDiskName("creator_exception"), [&](const DB::DisksMap &) -> DB::DiskPtr + { + throw ExceptionWithoutRethrow(DB::ErrorCodes::BAD_ARGUMENTS, "creation failed"); + }), + ExceptionWithoutRethrow); +} + +/// While a disk is being created, resolving it by name says so instead of reporting an unknown disk. +TEST_F(CASStartupStaleMounts, LookupOfADiskBeingCreatedSaysSo) +{ + const String name = uniqueDiskName("lookup_in_flight"); + const fs::path custom_disk_path = root / "custom_disk"; + fs::create_directories(custom_disk_path); + + String get_disk_message; + String policy_message; + int get_disk_code = 0; + getContext().context->getOrCreateDisk(name, [&](const DB::DisksMap &) -> DB::DiskPtr + { + try + { + getContext().context->getDisk(name); + } + catch (const DB::Exception & e) + { + get_disk_code = e.code(); + get_disk_message = e.message(); + } + try + { + getContext().context->getStoragePolicyFromDisk(name); + } + catch (const DB::Exception & e) + { + policy_message = e.message(); + } + return std::make_shared(name, custom_disk_path.string() + "/"); + }); + + EXPECT_EQ(get_disk_code, DB::ErrorCodes::UNKNOWN_DISK); + EXPECT_TRUE(get_disk_message.contains("being created")) << get_disk_message; + EXPECT_TRUE(policy_message.contains("being created")) << policy_message; + EXPECT_NO_THROW(getContext().context->getDisk(name)); +} + +/// A waiter of a failed creation gets a copy of the creator's exception of the same type. +TEST_F(CASStartupStaleMounts, WaiterOfAFailedCreationGetsTheExceptionType) +{ + const String name = uniqueDiskName("fenced_waiter"); + std::mutex mutex; + std::condition_variable cv; + bool creating = false; + bool creator_started = false; + bool waiter_got_fenced = false; + bool creator_got_fenced = false; + + std::thread creator([&] + { + try + { + getContext().context->getOrCreateDisk(name, [&](const DB::DisksMap &) -> DB::DiskPtr + { + { + std::lock_guard lock(mutex); + creating = true; + cv.notify_all(); + } + waitForWaiters(name, 1); + throw MountFencedException("fenced"); + }); + } + catch (const MountFencedException &) + { + creator_got_fenced = true; + } + catch (...) + { + } + }); + { + std::unique_lock lock(mutex); + creator_started = cv.wait_for(lock, std::chrono::seconds(30), [&] { return creating; }); + } + if (creator_started) + { + try + { + getContext().context->getOrCreateDisk(name, [](const DB::DisksMap &) -> DB::DiskPtr { return nullptr; }); + } + catch (const MountFencedException &) + { + waiter_got_fenced = true; + } + catch (...) + { + } + } + creator.join(); + + EXPECT_TRUE(creator_started) << "the creator never started"; + EXPECT_TRUE(creator_got_fenced); + EXPECT_TRUE(waiter_got_fenced); +} diff --git a/src/Disks/tests/gtest_cas_upload_detached.cpp b/src/Disks/tests/gtest_cas_upload_detached.cpp index 21c8ca69e5d92..358d1da58e51e 100644 --- a/src/Disks/tests/gtest_cas_upload_detached.cpp +++ b/src/Disks/tests/gtest_cas_upload_detached.cpp @@ -230,25 +230,23 @@ TEST(CASUploadDetached, ExistingCleanHeadsAndObservesWithoutPublication) EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::CASBlobBodyPutAvoided], avoided_before + 1); } -TEST(CASUploadDetached, ExistingBodyWithoutMetadataBackfillsWithoutPublication) +/// A body without a marker is never adopted: it is republished once and the marker is created `Clean`. +TEST(CASUploadDetached, ExistingBodyWithoutMetadataRepublishesOnceThenClean) { - const String payload = "mandatory-head-metadata-backfill"; + const String payload = "mandatory-head-metadata-absent"; const BlobRef ref = idOf(payload); auto backend = std::make_shared(); auto store = openUploadPool(backend); seedPresentBody(*backend, store->layout(), store->poolMeta(), ref, payload); - auto build = precommitBuildFor(store, RootNamespace{"srv1/protocol-backfill"}, "part", payload); + auto build = precommitBuildFor(store, RootNamespace{"srv1/protocol-absent-meta"}, "part", payload); backend->watch(store->layout().blobKey(ref), store->layout().blobMetaKey(ref)); const BlobUploadResult result = build->uploadBlobDetached( BlobUploadRequest{ref, BlobSource::fromString(payload), payload.size()}); EXPECT_EQ(result.dep.proof, BlobDependencyProof::Materialized); - ASSERT_FALSE(backend->operations.empty()); - EXPECT_EQ(backend->operations.front(), "head"); - EXPECT_EQ(backend->blob_heads, 1u); - EXPECT_EQ(backend->meta_gets, 1u); - EXPECT_EQ(backend->publish_calls, 0u); + EXPECT_EQ(result.diagnostics.action, BlobMaterializationAction::Published); + EXPECT_EQ(backend->operations, (std::vector{"head", "meta-get", "publish"})); const auto meta = loadMetaForTest(*backend, store->layout(), u128Of(payload)); ASSERT_TRUE(meta.has_value()); EXPECT_EQ(meta->meta.state, MetaState::Clean); @@ -312,8 +310,8 @@ TEST(CASUploadDetached, PresentCondemnedPublishesFreshAndQueuedOldDeleteMisses) } } -/// A present body with absent metadata is observed and backfilled `Clean` without publication. -TEST(CASUploadDetached, PresentBodyWithoutMetadataBackfills) +/// A present body with absent metadata is republished as if absent, and the marker is created `Clean`. +TEST(CASUploadDetached, PresentBodyWithoutMetadataRepublishesThenClean) { const RootNamespace ns{"srv1/nsAdopt"}; const String ref_name = "part"; @@ -324,7 +322,7 @@ TEST(CASUploadDetached, PresentBodyWithoutMetadataBackfills) { b = std::make_shared(); s = openUploadPool(b); - seedPresentBody(*b, s->layout(), s->poolMeta(), blob, payload); /// body present, no meta: backfill + seedPresentBody(*b, s->layout(), s->poolMeta(), blob, payload); /// body present, no meta build = precommitBuildFor(s, ns, ref_name, payload); }; @@ -340,14 +338,14 @@ TEST(CASUploadDetached, PresentBodyWithoutMetadataBackfills) const BlobUploadResult r = build1->uploadBlobDetached( BlobUploadRequest{blob, BlobSource::fromString(payload), payload.size()}); - EXPECT_EQ(r.diagnostics.action, BlobMaterializationAction::Observed); - EXPECT_EQ(r.diagnostics.reason, std::nullopt); - EXPECT_EQ(r.diagnostics.transport, std::nullopt); + EXPECT_EQ(r.diagnostics.action, BlobMaterializationAction::Published); + EXPECT_EQ(r.diagnostics.reason, std::optional(BlobPublicationReason::Absent)); + EXPECT_EQ(r.diagnostics.transport, std::optional(BlobPublicationTransport::Streaming)); EXPECT_EQ(r.dep.proof, BlobDependencyProof::Materialized); EXPECT_EQ(r.dep.size, payload.size()); EXPECT_EQ(build1->dependencyProof(blob), std::nullopt); - /// The point-read backfilled a Clean meta. + /// The publication's reconcile created a Clean meta. EXPECT_EQ(metaStateAt(*b1, s1->layout(), payload), std::optional(MetaState::Clean)); std::shared_ptr b2; diff --git a/src/Disks/tests/gtest_cas_upload_fanout.cpp b/src/Disks/tests/gtest_cas_upload_fanout.cpp index 5989ea44ccd30..8728fe38b5e02 100644 --- a/src/Disks/tests/gtest_cas_upload_fanout.cpp +++ b/src/Disks/tests/gtest_cas_upload_fanout.cpp @@ -320,8 +320,8 @@ TEST(CASUploadFanout, CopiedAndMovedRequestsSharePublicationAttemptedState) /// Test 1 (spec §1 "serial-vs-parallel equivalence for successful runs"): a multi-blob part that /// exercises every branch of `uploadBlobDetached` produces IDENTICAL recorded deps and IDENTICAL backend /// end state whether the fan-out runs serially (pool size 1) or in parallel (pool size 4). It covers -/// present-clean observation, metadata backfill, fresh local publication, staging copy, and local and -/// staged condemned-body republication. +/// present-clean observation, republication of a body without metadata, fresh local publication, +/// staging copy, and local and staged condemned-body republication. namespace { @@ -337,7 +337,7 @@ struct WorldA }; const char * const kObserved = "fanoutA-observed-clean"; -const char * const kAdopt = "fanoutA-head-miss-adopt"; +const char * const kAbsentMarker = "fanoutA-absent-marker"; const char * const kFresh = "fanoutA-fresh-local"; const char * const kStaging = "fanoutA-s3-staging"; const char * const kResLocal = "fanoutA-condemned-local"; @@ -355,8 +355,8 @@ WorldA arrangeWorldA() seedPresentBody(*w.b, w.s->layout(), w.s->poolMeta(), kObserved); writeMetaClean(*w.b, w.s->layout(), u128Of(kObserved), std::string(kObserved).size()); - /// HEAD-miss then 412-path live adopt with meta backfill: present body, NO meta, not cached. - seedPresentBody(*w.b, w.s->layout(), w.s->poolMeta(), kAdopt); + /// Present body, NO meta: republished like an absent body. + seedPresentBody(*w.b, w.s->layout(), w.s->poolMeta(), kAbsentMarker); /// fresh local streaming: nothing present. @@ -389,13 +389,13 @@ WorldA arrangeWorldA() w.requests = { localRequest(kObserved), - localRequest(kAdopt), + localRequest(kAbsentMarker), localRequest(kFresh), s3Request(kStaging, "p/staging/mount1/A-staging.tmp"), localRequest(kResLocal), s3Request(kResS3, "p/staging/mount1/A-republish.tmp"), }; - w.payloads = {kObserved, kAdopt, kFresh, kStaging, kResLocal, kResS3}; + w.payloads = {kObserved, kAbsentMarker, kFresh, kStaging, kResLocal, kResS3}; return w; } diff --git a/src/Disks/tests/gtest_disk_access_check.cpp b/src/Disks/tests/gtest_disk_access_check.cpp new file mode 100644 index 0000000000000..d97c7a72d59ab --- /dev/null +++ b/src/Disks/tests/gtest_disk_access_check.cpp @@ -0,0 +1,64 @@ +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace +{ + +/// Counts the access probes running at the same time. A probe waits for a second one to arrive, so +/// two unserialized probes meet inside `checkAccessImpl` and a serialized pair runs one after the other. +struct ProbeMeter +{ + std::mutex mutex; + std::condition_variable arrived; + size_t active = 0; + size_t max_active = 0; +}; + +class ProbeDisk : public DB::DiskLocal +{ +public: + ProbeDisk(const String & name_, const String & path_, ProbeMeter & meter_) : DiskLocal(name_, path_), meter(meter_) { } + +protected: + void checkAccessImpl(const String &) override + { + std::unique_lock lock(meter.mutex); + ++meter.active; + meter.max_active = std::max(meter.max_active, meter.active); + meter.arrived.notify_all(); + meter.arrived.wait_for(lock, std::chrono::milliseconds(300), [&] { return meter.active == 2; }); + --meter.active; + } + +private: + ProbeMeter & meter; +}; + +} + +/// Every disk writes the probe `clickhouse_access_check_`; disks created in parallel +/// (`Context::getOrCreateDisk`) can share a backing store, so their probes must not overlap. +TEST(DiskAccessCheck, ProbesOfDisksOverOneDirectoryDoNotOverlap) +{ + DB::ServerUUID::setRandomForUnitTests(); + auto backing = createDisk("access_check"); + ProbeMeter meter; + auto a = std::make_shared("a", backing->getPath(), meter); + auto b = std::make_shared("b", backing->getPath(), meter); + + std::thread ta([&] { a->startup(/*skip_access_check=*/false); }); + std::thread tb([&] { b->startup(/*skip_access_check=*/false); }); + ta.join(); + tb.join(); + + EXPECT_EQ(meter.max_active, 1u); + destroyDisk(backing); +} diff --git a/src/Disks/tests/gtest_s3_object_creation_requests.cpp b/src/Disks/tests/gtest_s3_object_creation_requests.cpp new file mode 100644 index 0000000000000..a14195aaf3909 --- /dev/null +++ b/src/Disks/tests/gtest_s3_object_creation_requests.cpp @@ -0,0 +1,456 @@ +#include + +#include "config.h" + +#if USE_AWS_S3 + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +/// Per-verb wire counts of the requests `S3ObjectStorage` sends to create an object. + +namespace DB::ErrorCodes +{ +extern const int BAD_ARGUMENTS; +extern const int S3_ERROR; +} + +using CountingS3::CountingS3Server; +using CountingS3::Verb; +using S3ObjectCreationTestStorage::forceOnePartMultipart; +using S3ObjectCreationTestStorage::forceOnePartMultipartCopy; +using S3ObjectCreationTestStorage::makeRetryingStorage; +using S3ObjectCreationTestStorage::thrownCode; + +namespace +{ + +void copyObject(DB::S3ObjectStorage & storage, const DB::WriteSettings & write_settings) +{ + storage.copyObject(DB::StoredObject("stage"), DB::StoredObject("copy"), DB::ReadSettings{}, write_settings); +} + +void writeObject(DB::S3ObjectStorage & storage, const DB::WriteSettings & write_settings) +{ + auto buffer = storage.writeObject( + DB::StoredObject("object"), DB::WriteMode::Rewrite, {}, DB::DBMS_DEFAULT_BUFFER_SIZE, write_settings); + buffer->write(std::string(100, 'd').data(), 100); + buffer->finalize(); +} + +#ifdef DEBUG_OR_SANITIZER_BUILD +/// A copy that asks for a creation client but allows the read-write fallback; sends no request. +void copyWithCreationClientAllowingFallback() +{ + auto storage = makeRetryingStorage("http://127.0.0.1:1"); + auto client = storage->getS3StorageClient(); + DB::copyS3File( + client, "test-bucket", "stage", CountingS3::object_size, client, "test-bucket", "copy", + DB::S3Settings{}.request_settings, DB::ReadSettings{}, /*blob_storage_log=*/nullptr, /*schedule=*/{}, + /*fallback_file_reader=*/{}, /*object_metadata=*/std::nullopt, DB::ObjectStorageCopyMode::Default, + /*object_creation_client=*/client); +} +#endif + +/// Captures what any logger writes at `priority` and above while in scope. +class ScopedLogCapture +{ +public: + explicit ScopedLogCapture(int priority = Poco::Message::PRIO_ERROR) + : channel(new Poco::StreamChannel(stream)) + , root_channel(Poco::Logger::root().getChannel(), /*shared=*/true) + , root_level(Poco::Logger::root().getLevel()) + { + std::vector names; + Poco::Logger::names(names); + for (const auto & name : names) + if (auto * logger = Poco::Logger::has(name)) + saved.push_back({name, Poco::AutoPtr(logger->getChannel(), /*shared=*/true), logger->getLevel()}); + Poco::Logger::setChannel(Poco::Logger::ROOT, channel.get()); + Poco::Logger::setLevel(Poco::Logger::ROOT, priority); + } + + ~ScopedLogCapture() + { + /// A logger created in scope took the capture's channel and gets the root's. + Poco::Logger::setChannel(Poco::Logger::ROOT, root_channel.get()); + Poco::Logger::setLevel(Poco::Logger::ROOT, root_level); + for (auto & [name, old_channel, old_level] : saved) + { + if (auto * logger = Poco::Logger::has(name)) + { + logger->setChannel(old_channel.get()); + logger->setLevel(old_level); + } + } + } + + std::string captured() const { return stream.str(); } + +private: + struct SavedLogger + { + std::string name; + Poco::AutoPtr channel; + int level; + }; + + std::ostringstream stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + Poco::AutoPtr channel; + Poco::AutoPtr root_channel; + int root_level; + std::vector saved; +}; + +DB::WriteSettings singleAttemptCreation() +{ + DB::WriteSettings write_settings; + write_settings.object_storage_single_attempt_object_creation = true; + return write_settings; +} + +DB::WriteSettings singleAttemptNativeCopy() +{ + DB::WriteSettings write_settings = singleAttemptCreation(); + write_settings.object_storage_copy_mode = DB::ObjectStorageCopyMode::NativeOnly; + return write_settings; +} + +/// One way `S3ObjectStorage` creates an object, and the verb that creates it. +struct CreationPath +{ + std::string name; + Verb creating_verb; + void (*adjust_settings)(DB::S3Settings &); + bool is_copy; + + std::shared_ptr storage(const std::string & endpoint) const + { + return makeRetryingStorage(endpoint, adjust_settings ? adjust_settings : std::function{}); + } + + DB::WriteSettings singleAttemptSettings() const { return is_copy ? singleAttemptNativeCopy() : singleAttemptCreation(); } + + void create(DB::S3ObjectStorage & target, const DB::WriteSettings & write_settings) const + { + if (is_copy) + copyObject(target, write_settings); + else + writeObject(target, write_settings); + } +}; + +const CreationPath creation_paths[] = { + {"Put", Verb::PutObject, nullptr, false}, + {"Copy", Verb::CopyObject, nullptr, true}, + {"Complete", Verb::CompleteMultipartUpload, forceOnePartMultipart, false}, + {"MultipartCopyComplete", Verb::CompleteMultipartUpload, forceOnePartMultipartCopy, true}, +}; + +void PrintTo(const CreationPath & path, std::ostream * os) +{ + *os << path.name; +} + +class S3SingleAttemptObjectCreationPath : public ::testing::TestWithParam +{ +}; + +/// The paths that end in `CompleteMultipartUpload`. +class S3SingleAttemptMultipartCreationPath : public ::testing::TestWithParam +{ +}; + +} + +/// `writeObject` already applies the override; a copy reissues `CopyObject` in the same loop. +TEST(S3ObjectStorageCopyObject, HonoursUnexpectedWriteErrorRetryOverride) +{ + (void)getContext(); + CountingS3Server server(CountingS3::failing(Verb::CopyObject, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, "NoSuchKey")); + auto storage = makeRetryingStorage(server.getUrl()); + + EXPECT_EQ(thrownCode([&] { copyObject(*storage, DB::WriteSettings{}); }), DB::ErrorCodes::S3_ERROR); + EXPECT_EQ(server.count(Verb::CopyObject), 4u) << "the default `max_unexpected_write_error_retries`"; + + DB::WriteSettings write_settings; + write_settings.s3_max_unexpected_write_error_retries_override = 1; + EXPECT_EQ(thrownCode([&] { copyObject(*storage, write_settings); }), DB::ErrorCodes::S3_ERROR); + EXPECT_EQ(server.count(Verb::CopyObject), 5u); +} + +TEST_P(S3SingleAttemptObjectCreationPath, OneWireRequestOnLostResponse) +{ + (void)getContext(); + const CreationPath & path = GetParam(); + CountingS3Server server(CountingS3::dropping(path.creating_verb)); + + EXPECT_EQ(thrownCode([&] { path.create(*path.storage(server.getUrl()), path.singleAttemptSettings()); }), DB::ErrorCodes::S3_ERROR); + EXPECT_EQ(server.count(path.creating_verb), 1u); +} + +/// A writer that does not ask for single-attempt creation keeps the client's retries. +TEST_P(S3SingleAttemptObjectCreationPath, DefaultWriteStillResendsLostResponse) +{ + (void)getContext(); + const CreationPath & path = GetParam(); + CountingS3Server server(CountingS3::dropping(path.creating_verb, /*only_nth=*/1)); + + EXPECT_EQ(thrownCode([&] { path.create(*path.storage(server.getUrl()), DB::WriteSettings{}); }), 0); + EXPECT_EQ(server.count(path.creating_verb), 2u); +} + +/// The caller of a single-attempt creating request resolves the outcome and retries, so one failed +/// attempt is not an error for the operator. +TEST_P(S3SingleAttemptObjectCreationPath, LostResponseLogsNothingAtError) +{ + (void)getContext(); + const CreationPath & path = GetParam(); + CountingS3Server server(CountingS3::dropping(path.creating_verb)); + auto storage = path.storage(server.getUrl()); + + ScopedLogCapture capture; + EXPECT_EQ(thrownCode([&] { path.create(*storage, path.singleAttemptSettings()); }), DB::ErrorCodes::S3_ERROR); + EXPECT_EQ(capture.captured(), ""); + + /// Positive control: an `Error` message does reach the capture, so the emptiness above means something. + Poco::Logger::get("ScopedLogCaptureControl").error("control"); + EXPECT_NE(capture.captured().find("control"), std::string::npos); +} + +/// The flag alone bounds the `NO_SUCH_KEY` reissue loops above the client. +TEST_P(S3SingleAttemptObjectCreationPath, NoSuchKeyIsOneWireRequest) +{ + (void)getContext(); + const CreationPath & path = GetParam(); + CountingS3Server server(CountingS3::failing(path.creating_verb, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, "NoSuchKey")); + + EXPECT_EQ(thrownCode([&] { path.create(*path.storage(server.getUrl()), path.singleAttemptSettings()); }), DB::ErrorCodes::S3_ERROR); + EXPECT_EQ(server.count(path.creating_verb), 1u); +} + +INSTANTIATE_TEST_SUITE_P( + S3SingleAttemptObjectCreation, + S3SingleAttemptObjectCreationPath, + ::testing::ValuesIn(creation_paths), + [](const ::testing::TestParamInfo & param_info) { return param_info.param.name; }); + +/// After a lost single-attempt `CompleteMultipartUpload` the abort finds no upload: the completion may +/// have applied, and the caller resolves that. +TEST_P(S3SingleAttemptMultipartCreationPath, AbortAfterLostCompleteLogsNothingAtError) +{ + (void)getContext(); + const CreationPath & path = GetParam(); + CountingS3Server server([](const CountingS3::Request & request, Poco::Net::HTTPServerResponse & response) + { + if (request.verb == Verb::CompleteMultipartUpload) + throw CountingS3::DropResponse(); + if (request.verb == Verb::AbortMultipartUpload) + return CountingS3::sendError(response, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, "NoSuchUpload"); + CountingS3::answerOk(request.verb, response); + }); + auto storage = path.storage(server.getUrl()); + + ScopedLogCapture capture; + EXPECT_EQ(thrownCode([&] { path.create(*storage, path.singleAttemptSettings()); }), DB::ErrorCodes::S3_ERROR); + EXPECT_EQ(server.count(Verb::AbortMultipartUpload), 1u); + EXPECT_EQ(capture.captured(), ""); +} + +INSTANTIATE_TEST_SUITE_P( + S3SingleAttemptObjectCreation, + S3SingleAttemptMultipartCreationPath, + ::testing::Values(creation_paths[2], creation_paths[3]), + [](const ::testing::TestParamInfo & param_info) { return param_info.param.name; }); + +/// A `NO_SUCH_KEY` loop bounded to one request must not announce a retry. +TEST(S3SingleAttemptObjectCreation, LastNoSuchKeyAttemptDoesNotAnnounceARetry) +{ + (void)getContext(); + CountingS3Server server(CountingS3::failing(Verb::PutObject, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, "NoSuchKey")); + auto storage = makeRetryingStorage(server.getUrl()); + DB::WriteSettings write_settings; + write_settings.s3_max_unexpected_write_error_retries_override = 1; + + ScopedLogCapture capture(Poco::Message::PRIO_INFORMATION); + EXPECT_EQ(thrownCode([&] { writeObject(*storage, write_settings); }), DB::ErrorCodes::S3_ERROR); + EXPECT_NE(capture.captured().find("NO_SUCH_KEY"), std::string::npos) << capture.captured(); + EXPECT_EQ(capture.captured().find("will retry"), std::string::npos) << capture.captured(); +} + +/// The client resends a request whose response body fails mid-read; the `PUT` was applied by then. +TEST(S3SingleAttemptObjectCreation, PutIsOneWireRequestOnBrokenResponseBody) +{ + (void)getContext(); + CountingS3Server server(CountingS3::faulting(Verb::PutObject, [](Poco::Net::HTTPServerResponse & response) + { + response.set("ETag", "\"put-etag\""); + response.setContentType("application/xml"); + response.setContentLength(4096); + response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); + response.send() << "" << std::flush; + throw CountingS3::DropResponse(); + })); + + EXPECT_EQ(thrownCode([&] { writeObject(*makeRetryingStorage(server.getUrl()), singleAttemptCreation()); }), DB::ErrorCodes::S3_ERROR); + EXPECT_EQ(server.count(Verb::PutObject), 1u); +} + +/// A part creates no object, so a throttled part keeps the client's retries. +TEST(S3SingleAttemptObjectCreation, UploadPartStillRetried) +{ + (void)getContext(); + CountingS3Server server(CountingS3::failing( + Verb::UploadPart, Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR, "InternalError", /*only_nth=*/1)); + + EXPECT_EQ(thrownCode([&] { writeObject(*makeRetryingStorage(server.getUrl(), forceOnePartMultipart), singleAttemptCreation()); }), 0); + EXPECT_EQ(server.count(Verb::UploadPart), 2u); + EXPECT_EQ(server.count(Verb::CompleteMultipartUpload), 1u); +} + +TEST(S3SingleAttemptObjectCreation, CreateMultipartUploadStillRetried) +{ + (void)getContext(); + CountingS3Server server(CountingS3::failing( + Verb::CreateMultipartUpload, Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR, "InternalError", /*only_nth=*/1)); + + EXPECT_EQ(thrownCode([&] { writeObject(*makeRetryingStorage(server.getUrl(), forceOnePartMultipart), singleAttemptCreation()); }), 0); + EXPECT_EQ(server.count(Verb::CreateMultipartUpload), 2u); + EXPECT_EQ(server.count(Verb::CompleteMultipartUpload), 1u); +} + +TEST(S3SingleAttemptObjectCreation, UploadPartCopyStillRetried) +{ + (void)getContext(); + CountingS3Server server(CountingS3::failing( + Verb::UploadPartCopy, Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR, "InternalError", /*only_nth=*/1)); + + EXPECT_EQ( + thrownCode([&] { copyObject(*makeRetryingStorage(server.getUrl(), forceOnePartMultipartCopy), singleAttemptNativeCopy()); }), + 0); + EXPECT_EQ(server.count(Verb::UploadPartCopy), 2u); + EXPECT_EQ(server.count(Verb::CompleteMultipartUpload), 1u); +} + +/// A definite refusal means the `CopyObject` was not applied, so the multipart copy is the only creation. +TEST(S3SingleAttemptObjectCreation, CopyRefusedAsTooLargeBecomesOneMultipartCopy) +{ + (void)getContext(); + CountingS3Server server(CountingS3::failing(Verb::CopyObject, Poco::Net::HTTPResponse::HTTP_BAD_REQUEST, "EntityTooLarge")); + + EXPECT_EQ(thrownCode([&] { copyObject(*makeRetryingStorage(server.getUrl()), singleAttemptNativeCopy()); }), 0); + EXPECT_EQ(server.count(Verb::CopyObject), 1u); + EXPECT_EQ(server.count(Verb::CreateMultipartUpload), 1u); + EXPECT_EQ(server.count(Verb::CompleteMultipartUpload), 1u); +} + +/// The creating request has no second attempt, so its client must not use the short first-attempt timeouts. +TEST(S3SingleAttemptObjectCreation, CreationClientDisablesAdaptiveTimeouts) +{ + auto storage = makeRetryingStorage("http://127.0.0.1:1", {}, /*adaptive_timeouts=*/true); + ASSERT_TRUE(storage->getS3StorageClient()->getClientConfiguration().s3_use_adaptive_timeouts); + + EXPECT_FALSE(storage->getSingleAttemptClient(0, 0, /*disable_adaptive_timeouts=*/true) + ->getClientConfiguration().s3_use_adaptive_timeouts); + EXPECT_TRUE(storage->getSingleAttemptClient(0, 0)->getClientConfiguration().s3_use_adaptive_timeouts) + << "the control-plane clone keeps the disk's adaptive timeouts"; +} + +/// End to end: `writeObject` sends the `PUT` through that client. The delay is above the adaptive +/// first-attempt receive timeout of a `PUT` and below the configured one. +TEST(S3SingleAttemptObjectCreation, PutSlowerThanTheFirstAttemptTimeoutSucceedsOnce) +{ + (void)getContext(); + CountingS3Server server(CountingS3::faulting(Verb::PutObject, [](Poco::Net::HTTPServerResponse & response) + { + std::this_thread::sleep_for(std::chrono::milliseconds(3500)); + CountingS3::answerOk(Verb::PutObject, response); + })); + + EXPECT_EQ( + thrownCode([&] { writeObject(*makeRetryingStorage(server.getUrl(), {}, /*adaptive_timeouts=*/true), singleAttemptCreation()); }), + 0); + EXPECT_EQ(server.count(Verb::PutObject), 1u); +} + +/// The read-write fallback would write through the retrying client, so the storage refuses the request +/// before sending anything. +TEST(S3ObjectStorageCopyObject, SingleAttemptCreationRequiresNativeOnly) +{ + (void)getContext(); + CountingS3Server server([](const CountingS3::Request & request, Poco::Net::HTTPServerResponse & response) + { + CountingS3::answerOk(request.verb, response); + }); + + EXPECT_EQ( + thrownCode([&] { copyObject(*makeRetryingStorage(server.getUrl()), singleAttemptCreation()); }), + DB::ErrorCodes::BAD_ARGUMENTS); + EXPECT_EQ(server.count(Verb::HeadObject), 0u); + EXPECT_EQ(server.count(Verb::CopyObject), 0u); +} + +/// The S3-to-S3 shortcut creates the destination object, so it honours the flag like `copyObject`. +TEST(S3ObjectStorageCopyObject, CopyToAnotherStorageIsOneWireRequestWhenSingleAttempt) +{ + (void)getContext(); + const auto copy_to_another = [](DB::S3ObjectStorage & from, DB::S3ObjectStorage & to, const DB::WriteSettings & write_settings) + { + from.copyObjectToAnotherObjectStorage( + DB::StoredObject("stage"), DB::StoredObject("copy"), DB::ReadSettings{}, write_settings, to); + }; + + CountingS3Server dropping_server(CountingS3::dropping(Verb::CopyObject)); + auto dropping_storage = makeRetryingStorage(dropping_server.getUrl()); + EXPECT_EQ( + thrownCode([&] { copy_to_another(*dropping_storage, *dropping_storage, singleAttemptNativeCopy()); }), + DB::ErrorCodes::S3_ERROR); + EXPECT_EQ(dropping_server.count(Verb::CopyObject), 1u); + + CountingS3Server no_such_key_server(CountingS3::failing(Verb::CopyObject, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, "NoSuchKey")); + auto no_such_key_storage = makeRetryingStorage(no_such_key_server.getUrl()); + EXPECT_EQ( + thrownCode([&] { copy_to_another(*no_such_key_storage, *no_such_key_storage, singleAttemptNativeCopy()); }), + DB::ErrorCodes::S3_ERROR); + EXPECT_EQ(no_such_key_server.count(Verb::CopyObject), 1u); + + EXPECT_EQ( + thrownCode([&] { copy_to_another(*dropping_storage, *dropping_storage, singleAttemptCreation()); }), + DB::ErrorCodes::BAD_ARGUMENTS); +} + +/// `copyS3File` itself only asserts it; a failed assertion aborts debug and sanitizer builds. +#ifdef DEBUG_OR_SANITIZER_BUILD +TEST(S3SingleAttemptObjectCreationDeathTest, CreationClientRequiresNativeOnlyCopy) +{ + (void)getContext(); + EXPECT_DEATH( + { + copyWithCreationClientAllowingFallback(); + std::_Exit(0); + }, + "NativeOnly"); +} +#endif + +#endif diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index e28948af5097c..d051728b8781a 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -591,8 +591,13 @@ Model::CompleteMultipartUploadOutcome Client::CompleteMultipartUpload(CompleteMu const auto & key = request.GetKey(); const auto & bucket = request.GetBucket(); + /// A single-attempt client sent this request once, so `NoSuchUpload` cannot mean that its own earlier + /// attempt completed; an object at the key may come from another writer. + const bool single_attempt = usesSingleAttemptRetryStrategy(); + if (!outcome.IsSuccess() - && outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_UPLOAD) + && outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_UPLOAD + && !single_attempt) { auto check_request = HeadObjectRequest() .WithBucket(bucket) @@ -605,7 +610,8 @@ Model::CompleteMultipartUploadOutcome Client::CompleteMultipartUpload(CompleteMu outcome = Aws::S3::Model::CompleteMultipartUploadOutcome(Aws::S3::Model::CompleteMultipartUploadResult()); } - if (outcome.IsSuccess() && provider_type == ProviderType::GCS && client_settings.gcs_issue_compose_request) + /// The compose rewrites the object as a new generation, a second creating request; off by default. + if (outcome.IsSuccess() && provider_type == ProviderType::GCS && client_settings.gcs_issue_compose_request && !single_attempt) { /// For GCS we will try to compose object at the end, otherwise we cannot do a native copy /// for the object (e.g. for backups) diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp index b6d4f9ec83c81..76f2dbb2df575 100644 --- a/src/IO/S3/copyS3File.cpp +++ b/src/IO/S3/copyS3File.cpp @@ -18,10 +18,13 @@ #include #include +#include #include #include +#include + namespace ProfileEvents { @@ -88,8 +91,10 @@ namespace const std::optional & object_metadata_, ThreadPoolCallbackRunnerUnsafe schedule_, BlobStorageLogWriterPtr blob_storage_log_, - const LoggerPtr log_) + const LoggerPtr log_, + const std::shared_ptr & object_creation_client_ = nullptr) : client_ptr(client_ptr_) + , object_creation_client(object_creation_client_ ? object_creation_client_ : client_ptr_) , dest_bucket(dest_bucket_) , dest_key(dest_key_) , request_settings(request_settings_) @@ -106,6 +111,8 @@ namespace protected: std::shared_ptr client_ptr; + /// Sends the requests that create the destination object. + std::shared_ptr object_creation_client; const String & dest_bucket; const String & dest_key; const S3::S3RequestSettings & request_settings; @@ -212,7 +219,7 @@ namespace ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; - auto outcome = client_ptr->CompleteMultipartUpload(request); + auto outcome = object_creation_client->CompleteMultipartUpload(request); auto elapsed = watch.elapsedMicroseconds(); if (blob_storage_log) @@ -251,6 +258,11 @@ namespace abort_request.SetKey(dest_key); abort_request.SetUploadId(multipart_upload_id); + /// A lost single-attempt completion may have consumed the upload. + std::optional no_such_upload_is_expected; + if (object_creation_client->usesSingleAttemptRetryStrategy()) + no_such_upload_is_expected.emplace(); + Stopwatch watch; auto outcome = client_ptr->AbortMultipartUpload(abort_request); auto elapsed = watch.elapsedMicroseconds(); @@ -311,7 +323,12 @@ namespace } catch (...) { - tryLogCurrentException(log, fmt::format("While performing multipart upload of object {} in bucket {}", dest_key, dest_bucket)); + /// A caller that sends the completion once resolves a failure and retries itself, failed + /// `CreateMultipartUpload` and `UploadPartCopy` included; CAS logs each failed attempt. + tryLogCurrentException( + log, + fmt::format("While performing multipart upload of object {} in bucket {}", dest_key, dest_bucket), + object_creation_client->usesSingleAttemptRetryStrategy() ? LogsLevel::debug : LogsLevel::error); task_tracker.safeWaitAll(); @@ -629,7 +646,8 @@ namespace BlobStorageLogWriterPtr blob_storage_log_, std::function fallback_method_, bool is_ranged_copy_, - bool allow_fallback_ = true) + bool allow_fallback_, + const std::shared_ptr & object_creation_client_) : UploadHelper( client_ptr_, dest_bucket_, @@ -638,7 +656,8 @@ namespace object_metadata_, schedule_, blob_storage_log_, - getLogger("copyS3File")) + getLogger("copyS3File"), + object_creation_client_) , src_bucket(src_bucket_) , src_key(src_key_) , offset(src_offset_) @@ -740,7 +759,7 @@ namespace if (client_ptr->isClientForDisk()) ProfileEvents::increment(ProfileEvents::DiskS3CopyObject); - auto outcome = client_ptr->CopyObject(request); + auto outcome = object_creation_client->CopyObject(request); if (outcome.IsSuccess()) { LOG_TRACE( @@ -934,7 +953,8 @@ namespace const CreateReadBuffer & fallback_file_reader, const std::optional & object_metadata, bool is_ranged_copy, - ObjectStorageCopyMode copy_mode) + ObjectStorageCopyMode copy_mode, + std::shared_ptr object_creation_client) { if (!dest_s3_client) dest_s3_client = src_s3_client; @@ -982,7 +1002,8 @@ namespace blob_storage_log, std::move(fallback_method), is_ranged_copy, - /*allow_fallback=*/copy_mode == ObjectStorageCopyMode::Default}; + /*allow_fallback=*/copy_mode == ObjectStorageCopyMode::Default, + object_creation_client}; helper.performCopy(); } } @@ -1001,8 +1022,12 @@ void copyS3File( ThreadPoolCallbackRunnerUnsafe schedule, const CreateReadBuffer & fallback_file_reader, const std::optional & object_metadata, - ObjectStorageCopyMode copy_mode) + ObjectStorageCopyMode copy_mode, + std::shared_ptr object_creation_client) { + /// The read-write fallback would write through the retrying client. + chassert(!object_creation_client || copy_mode == ObjectStorageCopyMode::NativeOnly); + copyS3FileImpl( std::move(src_s3_client), src_bucket, @@ -1020,7 +1045,8 @@ void copyS3File( fallback_file_reader, object_metadata, /* is_ranged_copy= */ false, - copy_mode); + copy_mode, + std::move(object_creation_client)); } void copyS3FileRange( @@ -1057,7 +1083,8 @@ void copyS3FileRange( fallback_file_reader, object_metadata, /* is_ranged_copy= */ true, - ObjectStorageCopyMode::Default); + ObjectStorageCopyMode::Default, + /* object_creation_client= */ nullptr); } } diff --git a/src/IO/S3/copyS3File.h b/src/IO/S3/copyS3File.h index 5847889e933b7..9106070471bab 100644 --- a/src/IO/S3/copyS3File.h +++ b/src/IO/S3/copyS3File.h @@ -42,6 +42,9 @@ std::unique_ptr createS3UploadBody( /// /// `copy_mode = NativeOnly` forbids the client-side read-write fallback. If native copy is disabled /// or cannot complete, the failure is propagated instead. +/// +/// `object_creation_client`, when set, sends `CopyObject` and `CompleteMultipartUpload`; it requires +/// `NativeOnly`, because the read-write fallback would write through the retrying client. void copyS3File( std::shared_ptr src_s3_client, const String & src_bucket, @@ -56,7 +59,8 @@ void copyS3File( ThreadPoolCallbackRunnerUnsafe schedule, const CreateReadBuffer & fallback_file_reader, const std::optional & object_metadata = std::nullopt, - ObjectStorageCopyMode copy_mode = ObjectStorageCopyMode::Default); + ObjectStorageCopyMode copy_mode = ObjectStorageCopyMode::Default, + std::shared_ptr object_creation_client = nullptr); /// Copies exactly `[src_offset, src_offset + src_size)` of a LARGER source object of size `src_object_size`. /// diff --git a/src/IO/S3/tests/CountingS3Server.h b/src/IO/S3/tests/CountingS3Server.h new file mode 100644 index 0000000000000..a339b764c4a20 --- /dev/null +++ b/src/IO/S3/tests/CountingS3Server.h @@ -0,0 +1,354 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// A local HTTP server standing in for S3 that counts requests per verb, so a test can assert how many +/// times a request went on the wire. Path-style requests only. +namespace CountingS3 +{ + +enum class Verb : size_t +{ + PutObject, + CopyObject, + CreateMultipartUpload, + UploadPart, + UploadPartCopy, + CompleteMultipartUpload, + AbortMultipartUpload, + HeadObject, + GetObject, + ComposeObject, + Other, + Count, +}; + +inline Verb classify(const Poco::Net::HTTPServerRequest & request) +{ + const std::string query = Poco::URI(request.getURI()).getRawQuery(); + const bool has_upload_id = query.contains("uploadId="); + const bool has_part_number = query.contains("partNumber="); + const bool is_copy = request.has("x-amz-copy-source"); + const std::string & method = request.getMethod(); + + if (method == Poco::Net::HTTPRequest::HTTP_HEAD) + return Verb::HeadObject; + if (method == Poco::Net::HTTPRequest::HTTP_GET && !query.contains("list-type")) + return Verb::GetObject; + if (method == Poco::Net::HTTPRequest::HTTP_PUT && query.contains("compose")) + return Verb::ComposeObject; + if (method == Poco::Net::HTTPRequest::HTTP_PUT && has_part_number) + return is_copy ? Verb::UploadPartCopy : Verb::UploadPart; + if (method == Poco::Net::HTTPRequest::HTTP_PUT) + return is_copy ? Verb::CopyObject : Verb::PutObject; + if (method == Poco::Net::HTTPRequest::HTTP_POST && has_upload_id) + return Verb::CompleteMultipartUpload; + if (method == Poco::Net::HTTPRequest::HTTP_POST && query.contains("uploads")) + return Verb::CreateMultipartUpload; + if (method == Poco::Net::HTTPRequest::HTTP_DELETE && has_upload_id) + return Verb::AbortMultipartUpload; + return Verb::Other; +} + +/// Thrown by a responder to close the connection without an answer: the client sees a network error. +/// Whether the request was applied depends on the thrower; `ObjectStore` faults fire before the put. +struct DropResponse : std::runtime_error +{ + DropResponse() : std::runtime_error("test server drops the response") {} +}; + +using VerbCounts = std::array, static_cast(Verb::Count)>; + +/// One request as the server received it. +struct Request +{ + Verb verb; + /// The 1-based count of requests of this verb, this one included. + size_t nth; + /// `/bucket/key`. + std::string path; + std::string body; +}; + +using Responder = std::function; + +class CountingHandler : public Poco::Net::HTTPRequestHandler +{ + VerbCounts & counts; + const Responder & respond; + +public: + CountingHandler(VerbCounts & counts_, const Responder & respond_) : counts(counts_), respond(respond_) {} + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override + { + const Verb verb = classify(request); + /// Read the whole body first, so a dropped response follows a fully received request. + std::string body; + if (request.hasContentLength()) + Poco::StreamCopier::copyToString64(request.stream(), body); + const size_t nth = ++counts[static_cast(verb)]; + respond( + Request{ + .verb = verb, + .nth = nth, + .path = Poco::URI(request.getURI()).getPath(), + .body = std::move(body)}, + response); + } +}; + +class CountingHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory +{ + VerbCounts & counts; + const Responder & respond; + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override + { + return new CountingHandler(counts, respond); + } + +public: + CountingHandlerFactory(VerbCounts & counts_, const Responder & respond_) : counts(counts_), respond(respond_) {} +}; + +class CountingS3Server +{ + VerbCounts counts{}; + Responder respond; + std::unique_ptr server_socket; + Poco::ThreadPool thread_pool; + std::unique_ptr server; + +public: + explicit CountingS3Server(Responder respond_) + : respond(std::move(respond_)) + , server_socket(std::make_unique(0)) + , thread_pool("CountingS3") + , server(std::make_unique( + Poco::SharedPtr(new CountingHandlerFactory(counts, respond)), + thread_pool, + *server_socket, + Poco::AutoPtr(new Poco::Net::HTTPServerParams()))) + { + server->start(); + } + + ~CountingS3Server() + { + server->stop(); + thread_pool.joinAll(); + } + + std::string getUrl() const { return "http://127.0.0.1:" + std::to_string(server_socket->address().port()); } + size_t count(Verb verb) const { return counts[static_cast(verb)].load(); } +}; + +inline void sendXml(Poco::Net::HTTPServerResponse & response, Poco::Net::HTTPResponse::HTTPStatus status, const std::string & body) +{ + response.setStatus(status); + response.setContentType("application/xml"); + response.setContentLength(static_cast(body.size())); + response.send() << body; +} + +inline void sendError(Poco::Net::HTTPServerResponse & response, Poco::Net::HTTPResponse::HTTPStatus status, + const std::string & code, const std::string & message = "injected") +{ + sendXml(response, status, "" + code + "" + + message + ""); +} + +/// The size every `HEAD` reports. +inline constexpr size_t object_size = 64; + +/// A store on which every request succeeds. +inline void answerOk(Verb verb, Poco::Net::HTTPServerResponse & response) +{ + switch (verb) + { + case Verb::PutObject: + case Verb::UploadPart: + case Verb::ComposeObject: + response.set("ETag", "\"part-or-put-etag\""); + response.setContentLength(0); + response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); + response.send(); + return; + case Verb::CopyObject: + sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, + "\"copy-etag\"" + "2026-01-01T00:00:00.000Z"); + return; + case Verb::UploadPartCopy: + sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, + "\"copy-part-etag\"" + "2026-01-01T00:00:00.000Z"); + return; + case Verb::CreateMultipartUpload: + sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, + "test-bucket" + "keyupload-1"); + return; + case Verb::CompleteMultipartUpload: + sendXml(response, Poco::Net::HTTPResponse::HTTP_OK, + "test-bucket" + "key\"complete-etag\""); + return; + case Verb::AbortMultipartUpload: + response.setStatus(Poco::Net::HTTPResponse::HTTP_NO_CONTENT); + response.send(); + return; + case Verb::HeadObject: + response.set("ETag", "\"head-etag\""); + response.set("Last-Modified", "Thu, 01 Jan 2026 00:00:00 GMT"); + response.setContentLength(object_size); + response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); + response.send(); + return; + case Verb::GetObject: + case Verb::Other: + case Verb::Count: + break; + } + sendError(response, Poco::Net::HTTPResponse::HTTP_NOT_IMPLEMENTED, "NotImplemented"); +} + +using Fault = std::function; + +/// Answers requests of `faulted_verb` with `fault` (only the `only_nth`-th one, or every one when it is 0) +/// and everything else with `answerOk`. +inline Responder faulting(Verb faulted_verb, Fault fault, size_t only_nth = 0) +{ + return [faulted_verb, respond_with = std::move(fault), only_nth](const Request & request, Poco::Net::HTTPServerResponse & response) + { + if (request.verb == faulted_verb && (only_nth == 0 || request.nth == only_nth)) + return respond_with(response); + answerOk(request.verb, response); + }; +} + +/// Reads the whole request and closes the connection without an answer. +inline Responder dropping(Verb faulted_verb, size_t only_nth = 0) +{ + return faulting(faulted_verb, [](Poco::Net::HTTPServerResponse &) { throw DropResponse(); }, only_nth); +} + +inline Responder failing(Verb faulted_verb, Poco::Net::HTTPResponse::HTTPStatus status, std::string code, size_t only_nth = 0) +{ + return faulting( + faulted_verb, + [status, error_code = std::move(code)](Poco::Net::HTTPServerResponse & response) { sendError(response, status, error_code); }, + only_nth); +} + +/// An in-memory bucket behind a `CountingS3Server`: `PutObject` stores a body, `HeadObject` and +/// `GetObject` read it, every other verb gets `answerOk`. `fault` sees each request first and, by +/// returning true or throwing `DropResponse`, answers it instead; such a request is not applied. +class ObjectStore +{ +public: + using FaultFn = std::function; + + explicit ObjectStore(FaultFn fault_ = {}) : fault(std::move(fault_)) {} + + /// Must not outlive this store. + Responder responder() + { + return [this](const Request & request, Poco::Net::HTTPServerResponse & response) { respond(request, response); }; + } + + void put(const std::string & path, std::string body) + { + std::lock_guard lock(mutex); + objects[path] = std::move(body); + } + + std::optional get(const std::string & path) const + { + std::lock_guard lock(mutex); + if (auto it = objects.find(path); it != objects.end()) + return it->second; + return std::nullopt; + } + + /// Every request in arrival order, unapplied ones included. + std::vector journal() const + { + std::lock_guard lock(mutex); + return requests; + } + +private: + void respond(const Request & request, Poco::Net::HTTPServerResponse & response) + { + { + std::lock_guard lock(mutex); + requests.push_back(request); + } + if (fault && fault(request, response)) + return; + + switch (request.verb) + { + case Verb::PutObject: + put(request.path, request.body); + break; + case Verb::HeadObject: + case Verb::GetObject: + { + auto body = get(request.path); + if (!body && request.verb == Verb::HeadObject) + { + response.setStatus(Poco::Net::HTTPResponse::HTTP_NOT_FOUND); + response.setContentLength(0); + response.send(); + return; + } + if (!body) + return sendError(response, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, "NoSuchKey"); + response.set("ETag", "\"stored-etag\""); + response.set("Last-Modified", "Thu, 01 Jan 2026 00:00:00 GMT"); + response.setContentLength(static_cast(body->size())); + response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); + if (request.verb == Verb::HeadObject) + response.send(); + else + response.send() << *body; + return; + } + default: + break; + } + answerOk(request.verb, response); + } + + FaultFn fault; + mutable std::mutex mutex; + std::map objects; + std::vector requests; +}; + +} diff --git a/src/IO/S3/tests/gtest_s3_client_single_attempt_completion.cpp b/src/IO/S3/tests/gtest_s3_client_single_attempt_completion.cpp new file mode 100644 index 0000000000000..3e869d1142443 --- /dev/null +++ b/src/IO/S3/tests/gtest_s3_client_single_attempt_completion.cpp @@ -0,0 +1,130 @@ +#include + +#include "config.h" + +#if USE_AWS_S3 + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +/// `Client::CompleteMultipartUpload` on a single-attempt client: one request went on the wire, so an +/// object at the key after `NoSuchUpload` proves nothing about this upload, and a GCS compose would be +/// a second creating request. + +using CountingS3::CountingS3Server; +using CountingS3::Verb; + +namespace +{ + +/// `gcs` makes it a GCS client that composes after a multipart upload; the endpoint carries the GCS host +/// name in its path, which is all the provider detection looks at. +std::shared_ptr makeRetryingClient(const std::string & endpoint, bool gcs) +{ + /// The client configuration keeps a reference to the filter. + static const DB::RemoteHostFilter remote_host_filter; + DB::S3::PocoHTTPClientConfiguration cfg = DB::S3::ClientFactory::instance().createClientConfiguration( + "us-east-1", + remote_host_filter, + /* s3_max_redirects = */ 100, + DB::S3::PocoHTTPClientConfiguration::RetryStrategy{.max_retries = 2, .initial_delay_ms = 1, .max_delay_ms = 1}, + /* s3_slow_all_threads_after_network_error = */ false, + /* s3_slow_all_threads_after_retryable_error = */ false, + /* enable_s3_requests_logging = */ false, + /* for_disk_s3 = */ false, + /* opt_disk_name = */ {}, + /* request_throttler = */ {}); + cfg.endpointOverride = gcs ? endpoint + "/storage.googleapis.com" : endpoint; + cfg.s3_use_adaptive_timeouts = false; + /// A fresh connection per request: a pooled one can outlive this test's ephemeral-port server. + cfg.http_keep_alive_timeout = 0; + return DB::S3::ClientFactory::instance().create( + cfg, + DB::S3::ClientSettings{ + .use_virtual_addressing = false, + .disable_checksum = false, + .gcs_issue_compose_request = gcs, + .is_s3express_bucket = false, + }, + "ACCESS_KEY_ID", "SECRET_ACCESS_KEY", "", {}, {}, DB::S3::CredentialsConfiguration{}); +} + +std::unique_ptr singleAttemptClone(const DB::S3::Client & base) +{ + auto cfg = base.getClientConfiguration(); + cfg.retry_strategy.max_retries = 0; + cfg.retryStrategy = std::make_shared(); + return base.cloneWithConfigurationOverride(cfg); +} + +DB::S3::Model::CompleteMultipartUploadOutcome complete(const DB::S3::Client & client) +{ + DB::S3::CompleteMultipartUploadRequest request; + request.SetBucket("test-bucket"); + request.SetKey("key"); + request.SetUploadId("upload-1"); + Aws::S3::Model::CompletedMultipartUpload upload; + upload.AddParts(Aws::S3::Model::CompletedPart().WithETag("\"part-etag\"").WithPartNumber(1)); + request.SetMultipartUpload(upload); + return client.CompleteMultipartUpload(request); +} + +CountingS3::Responder answerNoSuchUpload() +{ + return CountingS3::failing(Verb::CompleteMultipartUpload, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, "NoSuchUpload"); +} + +} + +TEST(S3ClientSingleAttemptCompletion, NoSuchUploadWithObjectAtKeyIsAnError) +{ + CountingS3Server server(answerNoSuchUpload()); + auto client = singleAttemptClone(*makeRetryingClient(server.getUrl(), /*gcs=*/false)); + + const auto outcome = complete(*client); + ASSERT_FALSE(outcome.IsSuccess()); + EXPECT_EQ(outcome.GetError().GetErrorType(), Aws::S3::S3Errors::NO_SUCH_UPLOAD); + EXPECT_EQ(server.count(Verb::CompleteMultipartUpload), 1u); + EXPECT_EQ(server.count(Verb::HeadObject), 0u); +} + +/// A retrying client may have completed the upload on an attempt whose response it lost. +TEST(S3ClientSingleAttemptCompletion, OrdinaryClientReadsNoSuchUploadWithObjectAtKeyAsSuccess) +{ + CountingS3Server server(answerNoSuchUpload()); + auto client = makeRetryingClient(server.getUrl(), /*gcs=*/false); + + EXPECT_TRUE(complete(*client).IsSuccess()); + EXPECT_EQ(server.count(Verb::HeadObject), 1u); +} + +/// The compose rewrites the object as a new generation, a second creating request. +TEST(S3ClientSingleAttemptCompletion, GcsComposeIsSkipped) +{ + CountingS3Server server([](const CountingS3::Request & request, Poco::Net::HTTPServerResponse & response) + { + CountingS3::answerOk(request.verb, response); + }); + auto client = makeRetryingClient(server.getUrl(), /*gcs=*/true); + + /// The ordinary client composes, so this is a GCS client. + ASSERT_TRUE(complete(*client).IsSuccess()); + ASSERT_EQ(server.count(Verb::ComposeObject), 1u); + + EXPECT_TRUE(complete(*singleAttemptClone(*client)).IsSuccess()); + EXPECT_EQ(server.count(Verb::CompleteMultipartUpload), 2u); + EXPECT_EQ(server.count(Verb::ComposeObject), 1u); +} + +#endif diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index ef1dc613c1549..2b4be44f26c57 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -13,12 +13,14 @@ #include #include +#include #include #include #include #include #include +#include #include @@ -108,13 +110,15 @@ WriteBufferFromS3::WriteBufferFromS3( BlobStorageLogWriterPtr blob_log_, std::optional object_metadata_, ThreadPoolCallbackRunnerUnsafe schedule_, - const WriteSettings & write_settings_) + const WriteSettings & write_settings_, + std::shared_ptr object_creation_client_) : WriteBufferFromFileBase(std::min(buf_size_, static_cast(DBMS_DEFAULT_BUFFER_SIZE)), nullptr, 0) , bucket(bucket_) , key(key_) , request_settings(request_settings_) , write_settings(write_settings_) , client_ptr(std::move(client_ptr_)) + , object_creation_client(object_creation_client_ ? std::move(object_creation_client_) : client_ptr) , object_metadata(std::move(object_metadata_)) , buffer_allocation_policy(createBufferAllocationPolicy(request_settings)) , task_tracker( @@ -273,6 +277,18 @@ void WriteBufferFromS3::tryToAbortMultipartUpload() noexcept task_tracker->safeWaitAll(); abortMultipartUpload(); } + catch (const S3Exception & e) + { + /// A single-attempt `CompleteMultipartUpload` whose response was lost may have completed the + /// upload; the caller that sent it resolves the outcome. + if (e.getS3ErrorCode() == Aws::S3::S3Errors::NO_SUCH_UPLOAD && object_creation_client->usesSingleAttemptRetryStrategy()) + { + LOG_INFO(log, "Multipart upload to abort no longer exists. {}", getVerboseLogDetails()); + return; + } + LOG_ERROR(log, "Multipart upload hasn't been aborted. {}", getVerboseLogDetails()); + tryLogCurrentException(__PRETTY_FUNCTION__); + } catch (...) { LOG_ERROR(log, "Multipart upload hasn't been aborted. {}", getVerboseLogDetails()); @@ -488,6 +504,11 @@ void WriteBufferFromS3::abortMultipartUpload() if (client_ptr->isClientForDisk()) ProfileEvents::increment(ProfileEvents::DiskS3AbortMultipartUpload); + /// A lost single-attempt completion may have consumed the upload; `tryToAbortMultipartUpload` reports that. + std::optional no_such_upload_is_expected; + if (object_creation_client->usesSingleAttemptRetryStrategy()) + no_such_upload_is_expected.emplace(); + Stopwatch watch; auto outcome = client_ptr->AbortMultipartUpload(req); auto elapsed = watch.elapsedMicroseconds(); @@ -684,7 +705,7 @@ bool WriteBufferFromS3::completeMultipartUpload() ProfileEvents::increment(ProfileEvents::DiskS3CompleteMultipartUpload); Stopwatch watch; - auto outcome = client_ptr->CompleteMultipartUpload(req); + auto outcome = object_creation_client->CompleteMultipartUpload(req); auto elapsed = watch.elapsedMicroseconds(); ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, elapsed); @@ -708,8 +729,8 @@ bool WriteBufferFromS3::completeMultipartUpload() { last_error_type = error.GetErrorType(); last_error_details = error.GetExceptionName().empty() ? error.GetMessage() : error.GetExceptionName(); - LOG_INFO(log, "Multipart upload failed with a transient error ({}), will retry. {}, Parts: {}", - last_error_details, getVerboseLogDetails(), multipart_tags.size()); + LOG_INFO(log, "Multipart upload failed with a transient error ({}){}. {}, Parts: {}", + last_error_details, i + 1 < max_retry ? ", will retry" : "", getVerboseLogDetails(), multipart_tags.size()); } else { @@ -790,7 +811,7 @@ void WriteBufferFromS3::makeSinglepartUpload(WriteBufferFromS3::PartData && data CurrentThread::WriteThrottlingScope write_throttling_scope(write_settings.remote_throttler); Stopwatch watch; - auto outcome = client_ptr->PutObject(request); + auto outcome = object_creation_client->PutObject(request); auto elapsed = watch.elapsedMicroseconds(); ProfileEvents::increment(ProfileEvents::WriteBufferFromS3Microseconds, elapsed); @@ -812,16 +833,19 @@ void WriteBufferFromS3::makeSinglepartUpload(WriteBufferFromS3::PartData && data { /// For unknown reason, at least MinIO can respond with NO_SUCH_KEY for put requests - LOG_INFO(log, "Single part upload failed with NO_SUCH_KEY error. {}, size {}, will retry", getShortLogDetails(), content_length); + LOG_INFO(log, "Single part upload failed with NO_SUCH_KEY error. {}, size {}{}", + getShortLogDetails(), content_length, i + 1 < max_retry ? ", will retry" : ""); } else { /// Neither says anything to the operator: PreconditionFailed is an expected response for /// conditional writes (e.g. If-None-Match: *), handled by the caller (see - /// `S3::isPreconditionFailedError`); a SingleAttempt write is owned by an outer retry loop - /// that resolves the outcome and reissues, so its one failed attempt is not terminal either. + /// `S3::isPreconditionFailedError`); a request sent once (the SingleAttempt profile or a + /// single-attempt creation client) is owned by an outer retry loop that resolves the + /// outcome and reissues, so its one failed attempt is not terminal either. if (S3::isPreconditionFailedError(outcome.GetError()) - || write_settings.object_storage_retry_profile == ObjectStorageRetryProfile::SingleAttempt) + || write_settings.object_storage_retry_profile == ObjectStorageRetryProfile::SingleAttempt + || object_creation_client->usesSingleAttemptRetryStrategy()) LOG_DEBUG(log, "S3Exception name {}, Message: {}, bucket {}, key {}, object size {}", outcome.GetError().GetExceptionName(), outcome.GetError().GetMessage(), bucket, key, content_length); else diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index fbce538c0a431..5c54b4d50717e 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -44,7 +44,8 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase BlobStorageLogWriterPtr blob_log_, std::optional object_metadata_ = std::nullopt, ThreadPoolCallbackRunnerUnsafe schedule_ = {}, - const WriteSettings & write_settings_ = {}); + const WriteSettings & write_settings_ = {}, + std::shared_ptr object_creation_client_ = nullptr); ~WriteBufferFromS3() override; void nextImpl() override; @@ -91,6 +92,9 @@ class WriteBufferFromS3 final : public WriteBufferFromFileBase const S3::S3RequestSettings request_settings; const WriteSettings write_settings; const std::shared_ptr client_ptr; + /// Sends `PutObject` and `CompleteMultipartUpload`, the requests that create the object; + /// `client_ptr` unless the constructor got a separate one. + const std::shared_ptr object_creation_client; const std::optional object_metadata; /// Set from the PutObject / CompleteMultipartUpload response ETag on a successful upload; read /// by getResultObjectETag() after finalize(). Written by the upload worker, read after the diff --git a/src/IO/WriteSettings.h b/src/IO/WriteSettings.h index 708ec59402135..86cd11852f949 100644 --- a/src/IO/WriteSettings.h +++ b/src/IO/WriteSettings.h @@ -78,6 +78,12 @@ struct WriteSettings /// exactly one attempt. 0 = no override. size_t s3_max_unexpected_write_error_retries_override = 0; + /// Never resend the object-creating request (`PutObject`, `CopyObject`, `CompleteMultipartUpload`) after an + /// ambiguous outcome: it runs with adaptive timeouts off, and the `NO_SUCH_KEY` loops make one request. + /// Resends after a definite refusal and part retries stay. A copy must use `ObjectStorageCopyMode::NativeOnly`. + /// Honoured by `S3ObjectStorage` only. + bool object_storage_single_attempt_object_creation = false; + /// Selects the retry profile the object storage should execute this write under; see /// ObjectStorageRetryProfile. ObjectStorageRetryProfile object_storage_retry_profile = ObjectStorageRetryProfile::Default; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 93f4445ea4d92..2766158e3b153 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -719,6 +720,13 @@ struct ContextSharedPart : boost::noncopyable mutable std::unique_ptr compression_codec_selector; /// Storage disk chooser for MergeTree engines mutable std::shared_ptr merge_tree_disk_selector TSA_GUARDED_BY(storage_policies_mutex); + /// Custom disks being created by `getOrCreateDisk`, by name + struct DiskCreation + { + std::shared_future result; + size_t waiters = 0; + }; + mutable std::unordered_map disks_being_created TSA_GUARDED_BY(storage_policies_mutex); /// Storage policy chooser for MergeTree engines mutable std::shared_ptr merge_tree_storage_policy_selector TSA_GUARDED_BY(storage_policies_mutex); @@ -7524,27 +7532,118 @@ DiskPtr Context::getDisk(const String & name) const { std::lock_guard lock(shared->storage_policies_mutex); + return getDiskOrExplainCreation(lock, name); +} + +DiskPtr Context::getDiskOrExplainCreation(std::lock_guard & lock, const String & name) const TSA_REQUIRES(shared->storage_policies_mutex) +{ auto disk_selector = getDiskSelector(lock); + if (auto disk = disk_selector->tryGet(name)) + return disk; + + if (shared->disks_being_created.contains(name)) + throw Exception(ErrorCodes::UNKNOWN_DISK, "Disk {} is being created by another request, retry when it is created", backQuote(name)); return disk_selector->get(name); } +namespace +{ + +/// All waiters of one creation get the same stored exception. Each rethrows its own copy, so a +/// caller that modifies its exception (e.g. `addMessage`) does not race with the others. +DiskPtr getDiskCreationResult(const std::shared_future & creation) +{ + try + { + return creation.get(); + } + catch (...) + { + std::rethrow_exception(copyMutableException(std::current_exception())); + } +} + +} + DiskPtr Context::getOrCreateDisk(const String & name, DiskCreator creator) const { - std::lock_guard lock(shared->storage_policies_mutex); + std::promise promise; + std::shared_future creation; + bool is_creator = false; + DisksMap disks_map; + { + std::lock_guard lock(shared->storage_policies_mutex); - auto disk_selector = getDiskSelector(lock); + auto disk_selector = getDiskSelector(lock); + if (auto disk = disk_selector->tryGet(name)) + return disk; + + if (auto it = shared->disks_being_created.find(name); it != shared->disks_being_created.end()) + { + ++it->second.waiters; + creation = it->second.result; + } + else + { + disks_map = disk_selector->getDisksMap(); + creation = promise.get_future().share(); + shared->disks_being_created.emplace(name, ContextSharedPart::DiskCreation{creation, 0}); + is_creator = true; + } + } + + if (!is_creator) + return getDiskCreationResult(creation); + + /// A remote disk's startup can block on the network for a long time, so the creator runs without + /// `storage_policies_mutex`. Concurrent requests for the same name wait for this creation rather + /// than open the same storage a second time. + DiskPtr disk; + std::exception_ptr error; + try + { + disk = creator(disks_map); + } + catch (...) + { + error = std::current_exception(); + } - auto disk = disk_selector->tryGet(name); - if (!disk) { - disk = creator(getDisksMap(lock)); - const_cast(disk_selector.get())->addToDiskMap(name, disk); + std::lock_guard lock(shared->storage_policies_mutex); + shared->disks_being_created.erase(name); + if (!error) + { + try + { + const_cast(getDiskSelector(lock).get())->addToDiskMap(name, disk); + } + catch (...) + { + error = std::current_exception(); + } + } } + /// Waiters get a copy so that the creator can modify its own exception while they read theirs. + if (error) + promise.set_exception(copyMutableException(error)); + else + promise.set_value(disk); + + if (error) + std::rethrow_exception(error); return disk; } +size_t Context::getDiskCreationWaitersForTest(const String & name) const +{ + std::lock_guard lock(shared->storage_policies_mutex); + auto it = shared->disks_being_created.find(name); + return it == shared->disks_being_created.end() ? 0 : it->second.waiters; +} + StoragePolicyPtr Context::getStoragePolicy(const String & name) const { std::lock_guard lock(shared->storage_policies_mutex); @@ -7564,8 +7663,7 @@ StoragePolicyPtr Context::getStoragePolicyFromDisk(const String & disk_name) con if (!storage_policy) { - auto disk_selector = getDiskSelector(lock); - auto disk = disk_selector->get(disk_name); + auto disk = getDiskOrExplainCreation(lock, disk_name); auto volume = std::make_shared("_volume_" + disk_name, disk); static const auto move_factor_for_single_disk_volume = 0.0; @@ -7644,16 +7742,21 @@ void Context::updateStorageConfiguration(const Poco::Util::AbstractConfiguration { std::lock_guard lock(shared->storage_policies_mutex); Strings disks_to_reinit; + NameSet disks_being_created; + for (const auto & [name, _] : shared->disks_being_created) + disks_being_created.insert(name); if (shared->merge_tree_disk_selector) - shared->merge_tree_disk_selector - = shared->merge_tree_disk_selector->updateFromConfig(config, "storage_configuration.disks", shared_from_this()); + { + shared->merge_tree_disk_selector = shared->merge_tree_disk_selector->updateFromConfig( + config, "storage_configuration.disks", shared_from_this(), disks_being_created); + } if (shared->merge_tree_storage_policy_selector) { try { shared->merge_tree_storage_policy_selector = shared->merge_tree_storage_policy_selector->updateFromConfig( - config, "storage_configuration.policies", shared->merge_tree_disk_selector, disks_to_reinit); + config, "storage_configuration.policies", shared->merge_tree_disk_selector, disks_to_reinit, disks_being_created); } catch (Exception & e) { diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 1675118688599..93afbfed42416 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1849,7 +1849,14 @@ class Context: public ContextData, public std::enable_shared_from_this /// Provides storage disks DiskPtr getDisk(const String & name) const; using DiskCreator = std::function; + /// Returns the disk `name`, creating it with `creator` if it does not exist yet. The creator runs + /// without the storage lock and gets a snapshot of the disks. Concurrent calls for the same name + /// share one creation: the calling creator gets the disk or rethrows its own exception, the others + /// get the disk or a copy of the exception. A failed creation is not remembered. The creator must + /// not call `getOrCreateDisk` for its own name: it would wait for itself. DiskPtr getOrCreateDisk(const String & name, DiskCreator creator) const; + /// For tests: the number of requests waiting for a creation of the disk `name` started by another request. + size_t getDiskCreationWaitersForTest(const String & name) const; StoragePoliciesMap getPoliciesMap() const; DisksMap getDisksMap() const; @@ -2152,6 +2159,8 @@ class Context: public ContextData, public std::enable_shared_from_this StoragePolicySelectorPtr getStoragePolicySelector(std::lock_guard & lock) const; DiskSelectorPtr getDiskSelector(std::lock_guard & lock) const; + /// Like `DiskSelector::get`, but says so when the disk is still being created by `getOrCreateDisk`. + DiskPtr getDiskOrExplainCreation(std::lock_guard & lock, const String & name) const; DisksMap getDisksMap(std::lock_guard & lock) const; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 443d9d2d78a42..be76b6c659362 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -1212,8 +1212,9 @@ BlockIO InterpreterSystemQuery::execute() {"manifest_debris_removed", std::make_shared()}, {"staging_objects_removed", std::make_shared()}, {"mountpoint_objects_removed", std::make_shared()}, - {"slot_removed", std::make_shared()}, + {"member_retired", std::make_shared()}, {"warnings", std::make_shared()}, + {"namespaces_pending_gc", std::make_shared()}, }}; Block sample_block; for (const auto & column : columns) @@ -1229,8 +1230,9 @@ BlockIO InterpreterSystemQuery::execute() res_columns[i++]->insert(report.manifest_debris_removed); res_columns[i++]->insert(report.staging_objects_removed); res_columns[i++]->insert(report.mountpoint_objects_removed); - res_columns[i++]->insert(static_cast(report.slot_removed)); + res_columns[i++]->insert(static_cast(report.member_retired)); res_columns[i++]->insert(fmt::format("{}", fmt::join(report.warnings, "; "))); + res_columns[i++]->insert(report.namespaces_pending_gc); size_t num_rows = res_columns[0]->size(); auto source = std::make_shared(std::make_shared(std::move(sample_block)), Chunk(std::move(res_columns), num_rows)); diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index d750b1d94a610..c6864c2c9c561 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -560,7 +560,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & { /// SYSTEM CAS DROP POOL MEMBER FROM DISK [ON CLUSTER cluster]. /// Both the srid and the disk name are REQUIRED quoted string literals -- an srid is an - /// opaque server-root path (may contain '/'), not the bare identifier the sibling + /// opaque string, not the bare identifier the sibling /// CAS_* commands' disk TARGET accepts, so this does not go through /// parseQueryWithOnClusterAndTarget. ASTPtr ast; diff --git a/src/Storages/System/StorageSystemContentAddressedMounts.cpp b/src/Storages/System/StorageSystemContentAddressedMounts.cpp index ba8b8670cc105..b2ffa229a2901 100644 --- a/src/Storages/System/StorageSystemContentAddressedMounts.cpp +++ b/src/Storages/System/StorageSystemContentAddressedMounts.cpp @@ -48,15 +48,19 @@ StorageSystemContentAddressedMounts::StorageSystemContentAddressedMounts(const S {"expires_at", std::make_shared(3), "Time when the lease expires."}, {"min_active_build_sequence", std::make_shared(), "Oldest in-flight build sequence (UINT64_MAX means the mount said farewell)."}, {"gc_fenced", std::make_shared(), "1 if GC fenced this slot out (terminal)."}, - {"state", std::make_shared(), "Mount slot state: live, expired, terminated, fenced or corrupt."}, + {"state", std::make_shared(), "Mount slot state: live, expired, terminated, fenced, retired or corrupt."}, {"is_leader", std::make_shared(std::make_shared()), "1 if this server's GC scheduler holds this disk's leadership lease. NULL on rows describing other servers' mounts."}, {"pending_reclaim", std::make_shared(std::make_shared()), "Cumulative condemned-minus-deleted backlog observed by this process's GC on this disk. NULL on rows describing other servers' mounts."}, {"last_success_age_seconds", std::make_shared(std::make_shared()), "Seconds since this disk's GC last led a round (0 if it never led). NULL on rows describing other servers' mounts."}, {"wedged_namespace_count", std::make_shared(std::make_shared()), "Ref-append lanes currently wedged on this disk. NULL on rows describing other servers' mounts."}, {"lifecycle", std::make_shared(), "This server's content-addressed pool lifecycle for the disk (non-gated snapshot, always populated so a not-live disk stays visible): live, not_live, identity_lost, vanished, constructing (never started) or shutdown (torn down)."}, - {"lifecycle_reason", std::make_shared(), "The enum-clean sub-state word for a vanished disk: replaced or forgotten. Empty for every other lifecycle (so lifecycle || '(' || lifecycle_reason || ')' reads e.g. vanished(forgotten))."}, - {"lifecycle_detail", std::make_shared(), "The full typed reason text naming the actual cause when not live: the vanish diagnosis (data root replaced by a foreign pool / decommissioned by SYSTEM CAS FORGET at