Repository navigation
kv: fence raw applies and 2PC PREPAREs against the group watermark #1355
+1,432
−81
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,8 @@ | ||
| .PHONY: build test | ||
|
|
||
| build: | ||
| go build ./... | ||
|
|
||
| test: | ||
| go test -v -race ./... | ||
|
|
||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| package kv | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
|
|
||
| "github.com/bootjp/elastickv/store" | ||
| "github.com/cockroachdb/errors" | ||
| ) | ||
|
|
||
| // raftEncodeCommitTSFenceActivation turns on the raw and PREPARE apply-order | ||
| // fence for the group whose log carries it (docs/design/ | ||
| // 2026_09_26_partial_serializable_isolation_audit.md, "Raw and 2PC | ||
| // apply-order fence"). The entry has no payload. Its apply persists | ||
| // commitTSFenceEnabledKey, so the fence state is a function of the applied | ||
| // log and every replica of the group reaches the same verdict on every later | ||
| // raw entry and PREPARE. A per-node switch could not give that: replicas | ||
| // with different settings would accept and reject the same entry. | ||
| // | ||
| // 0x2f (field 5, wire type 7) is an invalid protobuf prefix, so it cannot be | ||
| // mistaken for a legacy bare Request (whose first byte is often 0x18, field 3 | ||
| // "ts" as a varint) or a RaftCommand. 0x1f belongs to guarded bootstrap. | ||
| const raftEncodeCommitTSFenceActivation byte = 0x2f | ||
|
|
||
| var ( | ||
| commitTSFenceEnabledKey = []byte(txnMetaPrefix + "commit_ts_fence/enabled") | ||
| commitTSFenceEnabledValue = []byte{1} | ||
| ) | ||
|
|
||
| var ErrInvalidCommitTSFenceActivation = errors.New("invalid commit-ts fence activation") | ||
|
|
||
| // EncodeCommitTSFenceActivation returns the Raft payload that turns the raw | ||
| // and PREPARE apply-order fence on for a group. | ||
| func EncodeCommitTSFenceActivation() []byte { | ||
| return []byte{raftEncodeCommitTSFenceActivation} | ||
| } | ||
|
|
||
| // CommitTSFenceEnabled reports whether st has applied the fence activation. | ||
| func CommitTSFenceEnabled(ctx context.Context, st store.MVCCStore) (bool, error) { | ||
| if st == nil { | ||
| return false, nil | ||
| } | ||
| value, err := st.GetAt(ctx, commitTSFenceEnabledKey, ^uint64(0)) | ||
| if errors.Is(err, store.ErrKeyNotFound) { | ||
| return false, nil | ||
| } | ||
| if err != nil { | ||
| return false, errors.WithStack(err) | ||
| } | ||
| if !bytes.Equal(value, commitTSFenceEnabledValue) { | ||
| return false, errors.Wrap(ErrInvalidCommitTSFenceActivation, "invalid marker value") | ||
| } | ||
| return true, nil | ||
| } | ||
|
|
||
| func (f *kvFSM) reloadCommitTSFence(ctx context.Context) error { | ||
| enabled, err := CommitTSFenceEnabled(ctx, f.store) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| f.commitTSFenceEnabled.Store(enabled) | ||
| return nil | ||
| } | ||
|
|
||
| // applyCommitTSFenceActivation persists the marker at the current watermark, | ||
| // like the prepare evidence activation, so it writes no version above any | ||
| // timestamp a client was handed. A replay finds the marker set and is a | ||
| // no-op. A failure to persist halts the apply loop: a replica that skipped | ||
| // the activation would keep accepting entries its peers fence. | ||
| func (f *kvFSM) applyCommitTSFenceActivation(ctx context.Context, body []byte) any { | ||
| if len(body) != 0 { | ||
| return errors.Wrapf(ErrInvalidCommitTSFenceActivation, "unexpected %d-byte payload", len(body)) | ||
| } | ||
| if f.commitTSFenceEnabled.Load() { | ||
| return nil | ||
| } | ||
| stamp := max(uint64(1), f.store.LastCommitTS()) | ||
| muts := []*store.KVPairMutation{{Op: store.OpTypePut, Key: commitTSFenceEnabledKey, Value: commitTSFenceEnabledValue}} | ||
| if err := f.store.ApplyMutationsRaftAt(ctx, muts, nil, stamp, stamp, f.pendingApplyIdx); err != nil { | ||
| return haltErr(errors.Wrap(errors.Mark(err, ErrRaftEntryApply), "kv/fsm: persist commit-ts fence activation")) | ||
| } | ||
| f.commitTSFenceEnabled.Store(true) | ||
| return nil | ||
| } | ||
|
|
||
| // checkPrepareCommitTSFence rejects a PREPARE whose settled commit timestamp | ||
| // is at or below the group's watermark, once the fence is active. A reader | ||
| // that finished before this PREPARE read at a snapshot no higher than that | ||
| // watermark; a reader after it meets the PREPARE's locks. Either way the | ||
| // COMMIT cannot land below a snapshot that was already read, which is why | ||
| // the COMMIT itself is not fenced. A transaction without a settled commit | ||
| // timestamp, or a PREPARE that writes nothing, is not checked. | ||
| func (f *kvFSM) checkPrepareCommitTSFence(meta TxnMeta, storeMuts []*store.KVPairMutation) error { | ||
| commitTS := prepareRouteFloorTimestamp(meta, 0) | ||
| if !f.commitTSFenceEnabled.Load() || commitTS == 0 { | ||
| return nil | ||
| } | ||
| if commitTS > f.store.LastCommitTS() { | ||
| return nil | ||
|
There was a problem hiding this comment.
 |
||
| } | ||
| for _, mut := range storeMuts { | ||
| if mut != nil && len(mut.Key) > 0 { | ||
| return errors.WithStack(store.NewWriteConflictError(mut.Key)) | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // applyRawMutations applies a raw entry's mutations, fenced once the | ||
| // activation has applied. Raw requests commit against the latest state, so | ||
| // commitTS is both the validation snapshot and the commit timestamp. | ||
| func (f *kvFSM) applyRawMutations(ctx context.Context, muts []*store.KVPairMutation, commitTS uint64) error { | ||
| if f.commitTSFenceEnabled.Load() { | ||
| return errors.WithStack(f.store.ApplyMutationsRaftAtFenced(ctx, muts, nil, commitTS, commitTS, f.pendingApplyIdx)) | ||
| } | ||
| return errors.WithStack(f.store.ApplyMutationsRaftAt(ctx, muts, nil, commitTS, commitTS, f.pendingApplyIdx)) | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
With
--commitTSFenceenabled, the acknowledged startup catalog path still makes this PREPARE verdict replica-dependent:setupDistributionCatalogcallsEnsureCatalogSnapshot, whoseCatalogStore.Savedirectly advances one replica'sLastCommitTS()without a Raft entry. A PREPARE whose settled timestamp falls between the replicas' watermarks is consequently rejected on that voter but installs locks on its peers, despite all applying the same log. Replicating the activation marker does not eliminate this divergence. Route catalog bootstrap through Raft, use a replicated-only watermark, or prevent activation until that prerequisite is satisfied; Draft status alone does not guard the executable flag.AGENTS.md reference: AGENTS.md:L35-L35
Useful? React with 👍 / 👎.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Addressed on HEAD 5bc688f: --commitTSFence now fails startup before stores open, and setupDistributionCatalog rejects any persisted activation marker before EnsureCatalogSnapshot, even when the flag is off. The fence cannot become active through this production startup path while direct catalog Save exists. PR #1364 will replace that Save with guarded Raft bootstrap before the gate can be removed. Root tests and full lint pass; the broader adapter suite timed out locally, so CI remains the verification gate.