Antalya 26.8: port the CAS PRs merged into antalya-26.6 after the PR 2540 cut

.github/workflows/master.yml

@@ -3345,7 +3345,7 @@ jobs:
           PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas s3 storage, parallel)' --workflow "MasterCI" --ci --timestamp
 
   stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2:
-    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g]
     needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }}
     name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)"
@@ -3396,7 +3396,7 @@ jobs:
           PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)' --workflow "MasterCI" --ci --timestamp
 
   stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2:
-    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g]
     needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }}
     name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)"

.github/workflows/pull_request.yml

@@ -2347,7 +2347,7 @@ jobs:
           PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas s3 storage, parallel)' --workflow "PR" --ci --timestamp
 
   stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2:
-    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g]
     needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }}
     name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)"
@@ -2396,7 +2396,7 @@ jobs:
           PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)' --workflow "PR" --ci --timestamp
 
   stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2:
-    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g]
     needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }}
     name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)"

.github/workflows/pull_request_community.yml

@@ -2537,7 +2537,7 @@ jobs:
           retention-days: 14
 
   stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2:
-    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g]
     needs: [build_amd_asan_ubsan, config_workflow, fast_test]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAxLzIp') }}
     name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 1/2)"
@@ -2601,7 +2601,7 @@ jobs:
           retention-days: 14
 
   stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2:
-    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c]
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g]
     needs: [build_amd_asan_ubsan, config_workflow, fast_test]
     if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYXNhbl91YnNhbiwgY2FzIHMzIHN0b3JhZ2UsIHBhcmFsbGVsLCAyLzIp') }}
     name: "Stateless tests (amd_asan_ubsan, cas s3 storage, parallel, 2/2)"

ci/defs/altinity_jobs.py

@@ -85,7 +85,15 @@ class AltinityJobConfigs:
         *[
             Job.ParamSet(
                 parameter=f"amd_asan_ubsan, cas s3 storage, parallel, {batch}/{total_batches}",
-                runs_on=RunnerLabels.AMD_MEDIUM_CPU,
+                # Under ASan the server RSS is mostly per-thread runtime state (fake
+                # stacks, thread stacks, allocator regions), not query memory, and it
+                # grows with the number of pool threads for the life of the process.
+                # On the 32 GiB runner a shard crosses the 0.8 memory-pressure
+                # threshold of clickhouse-test after ~2 h, and shedding workers does
+                # not lower RSS, so the shard drains to one worker and hits the job
+                # timeout. Same remedy as the plain ASan parallel lane: the LARGE
+                # runner has the same 16 vCPU and twice the RAM.
+                runs_on=RunnerLabels.AMD_LARGE,
                 requires=[ArtifactNames.CH_AMD_ASAN_UBSAN_GH],
             )
             for total_batches in (2,)

docs/en/antalya/cas/architecture/backend.md

@@ -34,7 +34,7 @@ different semantics: `PartWriteTxn::ensureBlobPresent` owns `HEAD`, freshness me
 `publishBlob` only moves the selected bytes.
 
 Writer readiness is represented by `BlobDependencyProof`, not by token presence. `Materialized`
-means the writer observed a present non-condemned body or completed publication and metadata
+means the writer observed a present body with a `Clean` marker or completed publication and metadata
 reconciliation. `TrustedManifest` means a durable source manifest proves the blob and requires no
 blob I/O. Pending state and writer tokens are not stored in the dependency record.
 
@@ -125,7 +125,8 @@ battery, skipping the battery used to skip it too, which is exactly why the thir
   writable generation-token mount; every other backend still skips only its permitted access-check
   I/O. This is what makes the exact-token delete check below unskippable on an *ordinary* writable
   mount. Decommissioning a pool member is the deliberate exception — it opens writable with the
-  battery skipped, because the fail-closed tradeoff inverts there: refusing an ordinary mount costs
+  battery skipped and logs one line naming what went unverified (versioning, conditional writes,
+  list-after-write), because the fail-closed tradeoff inverts there: refusing an ordinary mount costs
   availability and protects data, whereas refusing a decommission strands a pool with a dead replica
   in it and leaves the operator no way forward.
 - `checkConditionalWriteSingleAttemptSupport` — inside the battery. Refuses to mount writable unless the underlying
@@ -134,6 +135,14 @@ battery, skipping the battery used to skip it too, which is exactly why the thir
   committed, so retries on the conditional path must be explicit CAS state-machine transitions, not
   transparent client behavior.
 
+Blob publication is unconditional and does not use that profile, but its object-creating request
+(`PutObject`, `CopyObject`, `CompleteMultipartUpload`) is also sent once. On a store with
+content-derived ETags, a resent body that lands after GC deleted its incarnation brings that ETag
+back, and the pending exact-ETag delete then removes a live body. The request runs on a clone of the
+disk client with the disk's `request_timeout_ms`, no adaptive first-attempt timeouts, and the connect
+timeout capped as for the control plane. `CreateMultipartUpload`, `UploadPart` and `UploadPartCopy`
+keep the disk client's retries.
+
 A third staging check requires `supportsCopyMode(ObjectStorageCopyMode::NativeOnly)` when
 `cas_staging_backend = s3`. Writable mount fails closed when native same-store copy is unavailable; it
 does not silently fall back to local staging. Ordinary non-CAS `copyObject` fallback behavior is

docs/en/antalya/cas/architecture/blob-protocol.md

@@ -36,11 +36,11 @@ sequenceDiagram
     alt body present
         S3-->>Writer: present, size, backend token t1
         Writer->>S3: GET .meta (point read, body never streamed)
-        alt meta Clean or absent
+        alt meta Clean
             Writer->>Writer: record token-free BlobDependencyProof::Materialized (never adopt t1)
-        else meta Condemned
+        else meta Condemned or absent
             Writer->>S3: unconditional publish with fresh envelope
-            Writer->>S3: reconcile .meta to Clean
+            Writer->>S3: create or reconcile .meta to Clean
         end
     else body absent
         S3-->>Writer: absent
@@ -56,18 +56,29 @@ Ordered steps in `PartWriteTxn::ensureBlobPresent`:
    still live.
 2. **Mandatory observation.** Every physical materialization begins with one blob `HEAD`, regardless
    of size, provider, staging backend, or whether another writer probably uploaded the same hash.
-3. On a hit, the writer reads `.meta`. `Clean` or absent metadata permits adoption; the writer
-   records a `Materialized` dependency proof without retaining the observed token. `Condemned`
-   requires a new publication.
+3. On a hit, the writer reads `.meta`. Only `Clean` permits adoption; the writer records a
+   `Materialized` dependency proof without retaining the observed token. `Condemned` or absent
+   metadata requires a new publication. `GC` deletes `.meta` right after the body, so an absent
+   marker can follow a delete that landed between the `HEAD` and the read.
 4. On a miss, the writer does not read `.meta` before publication. It publishes its own payload
    unconditionally, then creates or reconciles `.meta` to `Clean`.
 5. Streaming publication mints a fresh `incarnation_tag` and can use ordinary multipart. The first
    publication of an S3-staged source may use native same-store copy only after a miss; a condemned
    or subsequent publication retags and streams the staged payload.
+   The request that creates the object (`PutObject`, `CopyObject`, `CompleteMultipartUpload`) is
+   never resent by the S3 client after an ambiguous outcome; resends after a definite refusal, such as
+   expired credentials or a redirect, stay. On a store with content-derived ETags, a resent
+   envelope that lands after GC deleted it would restore the condemned incarnation. Part uploads keep
+   the client's retries.
 6. `BlobSource::beginPublication` consumes the shared, monotonic `publication_attempted` state
    before backend I/O. A lost response cannot re-enable verbatim copy on a later attempt.
-7. Retryable or ambiguous failures restart at `HEAD`. No dependency proof is recorded until a
-   present non-condemned body was observed, or publication and metadata reconciliation completed.
+7. Retryable or ambiguous failures restart at `HEAD`: up to 8 attempts, each with a fresh envelope.
+   The engine starts no new attempt after the 90-second deadline, but an attempt already started can
+   run longer: each creating request waits up to the disk's `request_timeout_ms`, and multipart part
+   uploads keep their own retries. A write can therefore take longer than 90 seconds before it
+   fails with a retryable error. No dependency
+   proof is recorded until a present body with a `Clean` marker was observed, or publication and
+   metadata reconciliation completed.
 
 **Two writers uploading identical content** may both observe absence and both publish. The last
 physical incarnation wins, but the key proves that both payloads have the same logical identity and
@@ -151,7 +162,7 @@ sequenceDiagram
     alt writer republished
         S3-->>GC: TokenMismatch -- nothing deleted, blob is live at t2
     else genuinely dead
-        S3-->>GC: Deleted -- then drop the .meta only while it is still Condemned at round n or older
+        S3-->>GC: Deleted -- then drop the .meta only while it is still Condemned at round n+1 or older
     end
 ```
 
@@ -163,7 +174,7 @@ and a writer never trusts a body it did not itself just write.
 
 Why this closes the race in both directions:
 
-- A writer that **adopts** a present body must have read a non-`Condemned` marker, and its precommit
+- A writer that **adopts** a present body must have read a `Clean` marker, and its precommit
   edge was durable *before* that read. The next fold therefore sees in-degree ≥ 1 and spares the
   blob.
 - A writer that **replaces a condemned incarnation** changes the token. A stale `deleteExact(t1)` then returns
@@ -174,9 +185,11 @@ Why this closes the race in both directions:
   at the entry's round or newer. A `Clean`, absent, older-round or unreadable marker never
   graduates the entry: `GC` does not throw, it carries the entry and rewrites the marker at the
   current round.
-- The delete job removes the `.meta` only while it is still `Condemned` at the entry's round or
-  older, and conditionally on the version it read. A marker rewritten `Clean` by a writer, or
-  condemned again at a later round, stays.
+- The delete job removes the `.meta` only while it is still `Condemned` at a round before the
+  deleting round, and conditionally on the version it read. It also removes a marker the graduation
+  gate re-stamped. A marker rewritten `Clean` by a writer, or condemned in the deleting round,
+  stays. The delete job runs best effort: if it fails, the `Condemned` marker outlives its body
+  until a writer of that content replaces it, which costs storage, not safety.
 
 Both directions degrade to a spurious re-upload or a no-op delete. Neither can lose data or leave
 a dangling manifest entry.
@@ -189,8 +202,9 @@ in RAM at a time. Native object storage streams and can use multipart.
 
 `.meta` has exactly two states: `Clean` (body present, may be referenced) and `Condemned`
 (`GC` observed zero in-degree; the body is still present and a writer may replace it). An
-*absent* `.meta` reads exactly like `Clean` — there is no third "unaccounted" state in the
-stored format; `unaccounted` is a `cas-fsck` classification, not something `GC` ever writes.
+*absent* `.meta` never permits adoption: the writer republishes as for an absent body. There is
+no third "unaccounted" state in the stored format; `unaccounted` is a `cas-fsck` classification,
+not something `GC` ever writes.
 
 The record carries `state`, `condemn_round`, and `size`, and deliberately carries **no token**:
 it is a per-hash hint, not a per-incarnation fact. All safety comes from the body's in-envelope

docs/en/antalya/cas/architecture/correctness.md

@@ -24,7 +24,7 @@ The full model index (source `.tla` files and proof-run records) lives at
 
 | Model (`docs/superpowers/models/`) | Invariant it proves | Counterexample it caught |
 |---|---|---|
-| `CaBlobPublishCore.tla` | Split `HEAD` from unconditional publication while preserving logical content, fresh incarnation identity, monotonic `publication_attempted`, fence safety, and readiness only after metadata reconciliation | Eleven sabotage configurations cover condemned adoption, stale exact delete, ambiguous copy/PUT landing, envelope reuse after a later miss, missing precommit/meta reconciliation, fence loss, and wrong-content publication; three witnesses prove the safe paths are reachable |
+| `CaBlobPublishCore.tla` | Split `HEAD` from unconditional publication while preserving logical content, fresh incarnation identity, monotonic `publication_attempted`, fence safety, and readiness only after metadata reconciliation | Sabotage configurations cover condemned adoption, stale exact delete, ambiguous copy/PUT landing, envelope reuse after a later miss, missing precommit/meta reconciliation, fence loss, and wrong-content publication; three witnesses prove the safe paths are reachable. The model also lets the S3 client resend an identical envelope below the engine: on a content-ETag store that resend restores a condemned incarnation, and a stale exact delete then removes it, which is why blob creation is never resent |
 | `CaIncarnationCore.tla` | `INV_NO_DANGLE`, `INV_NO_LOSS`, `INV_NO_RETURN` — the safety spine for the whole GC core | `sab_unconddelete`: replacing the exact-token delete with an unconditional one lets a stale delete kill a replacement incarnation |
 | `CaBuildRootPrecommit.tla` | `INV_NO_DANGLE_COMMITTED` — a committed manifest never references an absent blob | Reproduces the dangling-manifest hazard exactly: `WriteBlob → AdoptBlob → BuildDie → GcDelete → Commit` with no presence re-check publishes a manifest over a deleted blob |
 | `CaGcLeaseCore.tla` | `NoFalseSteal` — no leader steals leadership from a live, mid-round incumbent | Without the advisory heartbeat, a frozen `seq` during a round looks identical to a dead leader, and a second leader steals from the alive one |