From 47dca8fed79e0f28b0ce78274543a4489a3727bf Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 13:42:13 +0000 Subject: [PATCH 01/72] pool: Cover detached volume recovery before commit. --- sandboxd/pool/volume_retirement_test.go | 58 +++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 sandboxd/pool/volume_retirement_test.go diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go new file mode 100644 index 00000000..fb8c542b --- /dev/null +++ b/sandboxd/pool/volume_retirement_test.go @@ -0,0 +1,58 @@ +package pool + +import ( + "errors" + "os" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestDetachedVolumeRetainsRecoveryProtectionUntilCommit(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detached result = %+v, %v", view, err) + } + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil || claims[sb.ID] == nil || len(claims[sb.ID].Volumes) != 1 { + t.Fatalf("failed write did not leave old recovery reference: %+v, %v", claims, err) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer admitted while old claim still references detached volume: %v", err) + } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { + t.Errorf("capture passed before recovery reference retired: %v", err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err = m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("restart lost old claim's reader exclusion: %v", err) + } + if _, err = m2.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}); err != nil { + t.Fatalf("retry after restart: %v", err) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("writer still blocked after successful retirement: %v", err) + } +} From be8e198c3d4d8e73b912b4597bbab75b7d26268c Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 13:52:42 +0000 Subject: [PATCH 02/72] pool: Define runtime volume registration requests. --- sandboxd/pool/pool.go | 5 +++++ sandboxd/types/api.go | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 9e0124cb..128d4492 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -118,6 +118,11 @@ var ( errNoEgressTap = errors.New("egress-lane claim has no lockable tap") ) +var ( + ErrVolumeRegistrationConflict = errors.New("volume registration conflicts with the catalog") + ErrVolumeCatalogUnavailable = errors.New("volume catalog requires a successful persistence retry") +) + // Engine is the slice of the cocoon driver the manager consumes. type Engine interface { Clone(ctx context.Context, fromDir, name string, key types.PoolKey) (types.VMRecord, error) diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index ac0cd6b4..1cee1753 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -22,6 +22,14 @@ type VolumeInfo struct { Writable bool `json:"writable,omitzero"` } +// VolumeRegistration binds an existing backing file to a runtime catalog name. +type VolumeRegistration struct { + Path string `json:"path"` + DirectIO string `json:"directio,omitempty"` + Writable bool `json:"writable,omitzero"` + Tenants []string `json:"tenants,omitempty"` +} + // VolumePatch is the wire body of PATCH /v1/sandboxes/{id}/volumes. type VolumePatch struct { Add []Volume `json:"add,omitzero"` From 8c4b2a40a713de8b99b7ded5ab8f2bce52e17a13 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 13:56:37 +0000 Subject: [PATCH 03/72] pool: Retain recoverable volume references until commit. --- sandboxd/pool/claims.go | 68 +++++++++-- sandboxd/pool/volume.go | 15 +++ sandboxd/pool/volume_mutation.go | 13 ++- sandboxd/pool/volume_retirement_test.go | 146 ++++++++++++++++++++++++ 4 files changed, 228 insertions(+), 14 deletions(-) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 5d7111df..9c3bb874 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -68,8 +68,10 @@ type claimDTO struct { // claimRow keeps its encoded member until set replaces the row; set never edits one. type claimRow struct { - dto claimDTO - raw []byte + id string + refs []types.Volume + dto claimDTO + raw []byte } // claimStore persists claimed sandboxes across daemon restarts; warm VMs are not persisted. @@ -81,10 +83,13 @@ type claimStore struct { writeMu sync.Mutex buf []byte - mu sync.Mutex - rows map[string]*claimRow - seq uint64 - written uint64 + mu sync.Mutex + rows map[string]*claimRow + seq uint64 + written uint64 + ackRows []*claimRow + attemptRows []*claimRow + replaceFile func([]byte) error recommitting atomic.Bool } @@ -110,6 +115,12 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { sb.VolumesWant = sb.Volumes } } + s.mu.Lock() + s.ackRows = make([]*claimRow, 0, len(claims)) + for _, sb := range claims { + s.ackRows = append(s.ackRows, newClaimRow(sb)) + } + s.mu.Unlock() return claims, nil } @@ -118,7 +129,7 @@ func (s *claimStore) set(sbs ...*types.Sandbox) claimSnapshot { s.mu.Lock() defer s.mu.Unlock() for _, sb := range sbs { - s.rows[sb.ID] = &claimRow{dto: dtoOf(sb)} + s.rows[sb.ID] = newClaimRow(sb) } s.seq++ return claimSnapshot{seq: s.seq} @@ -154,7 +165,7 @@ func (s *claimStore) reset(claims map[string]*types.Sandbox) claimSnapshot { defer s.mu.Unlock() s.rows = make(map[string]*claimRow, len(claims)) for id, sb := range claims { - s.rows[id] = &claimRow{dto: dtoOf(sb)} + s.rows[id] = newClaimRow(sb) } s.seq++ return claimSnapshot{seq: s.seq} @@ -186,10 +197,22 @@ func (s *claimStore) commit(snap claimSnapshot) error { s.buf = append(s.buf, row.raw...) } s.buf = append(s.buf, '}') - if err := s.replace(s.buf); err != nil { + s.mu.Lock() + for _, row := range rows { + if !slices.Contains(s.attemptRows, row) { + s.attemptRows = append(s.attemptRows, row) + } + } + s.mu.Unlock() + replace := s.replace + if s.replaceFile != nil { + replace = s.replaceFile + } + if err := replace(s.buf); err != nil { return err } s.mu.Lock() + s.ackRows, s.attemptRows = rows, nil s.written = seq s.mu.Unlock() return nil @@ -279,3 +302,30 @@ func volumesWantDTO(sb *types.Sandbox) []types.Volume { } return append([]types.Volume{}, sb.VolumesWant...) } + +func newClaimRow(sb *types.Sandbox) *claimRow { + refs := slices.Clone(sb.Volumes) + for _, v := range sb.VolumesWant { + if !slices.Contains(refs, v) { + refs = append(refs, v) + } + } + return &claimRow{id: sb.ID, refs: refs, dto: dtoOf(sb)} +} + +// recoveryVolumes includes every reference a failed replacement may have installed. +func (s *claimStore) recoveryVolumes() map[string][]types.Volume { + s.mu.Lock() + defer s.mu.Unlock() + refs := map[string][]types.Volume{} + for _, rows := range [][]*claimRow{s.ackRows, s.attemptRows} { + for _, row := range rows { + for _, v := range row.refs { + if !slices.Contains(refs[row.id], v) { + refs[row.id] = append(refs[row.id], v) + } + } + } + } + return refs +} diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 5a1e8eed..12810904 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -277,7 +277,22 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { // reserveVolumes admits one claim's volumes atomically; callers hold m.mu. func (m *Manager) reserveVolumes(volumes []types.Volume) error { + return m.reserveVolumesFor(volumes, nil) +} + +func (m *Manager) reserveVolumesFor(volumes []types.Volume, owner *types.Sandbox) error { + recovery := m.store.recoveryVolumes() for _, volume := range volumes { + for id, refs := range recovery { + for _, ref := range refs { + if owner != nil && m.claimed[id] == owner && owner.ID == id && ref == volume { + continue + } + if ref.Name == volume.Name && (ref.RW() || volume.RW()) { + return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) + } + } + } holders := m.volumeAdmission[volume.Name] if holders.writers > 0 || (volume.RW() && holders.readers > 0) { return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 33dc50e2..bd9a70ad 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -86,16 +86,17 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v }() m.mu.Lock() current, held := slices.Clone(sb.VolumesWant), heldVolumes(sb) + recovery := m.store.recoveryVolumes()[sb.ID] m.mu.Unlock() target, err := change(current) if err != nil { return view, err } - target, err = normalizeLiveVolumes(target, held) + target, err = normalizeLiveVolumes(target, slices.Concat(held, recovery)) if err != nil { return view, err } - if err = validateLiveTarget(held, target); err != nil { + if err = validateLiveTarget(slices.Concat(held, recovery), target); err != nil { return view, err } additions := volumeDifference(target, held) @@ -267,7 +268,7 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run m.mu.Unlock() return claimSnapshot{}, false, ErrVolumeCapture } - if err := m.reserveVolumes(additions); err != nil { + if err := m.reserveVolumesFor(additions, sb); err != nil { m.mu.Unlock() return claimSnapshot{}, false, err } @@ -345,7 +346,7 @@ func (m *Manager) ownsRunLocked(sb *types.Sandbox, run *volumeRun) bool { func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { run := m.volumeRuns[sb] - return hasVolumes(sb) || run != nil && len(run.holds) > 0 + return hasVolumes(sb) || run != nil && len(run.holds) > 0 || len(m.store.recoveryVolumes()[sb.ID]) > 0 } func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { @@ -355,7 +356,9 @@ func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { States: []types.VolumeState{}, Converging: m.volumeRuns[sb] != nil, } - for _, v := range heldVolumes(sb) { + held := heldVolumes(sb) + held = append(held, volumeDifference(m.store.recoveryVolumes()[sb.ID], held)...) + for _, v := range held { state := types.VolumeAttached if volumeIndex(sb.VolumesWant, v.Name) < 0 { state = types.VolumeDetaching diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go index fb8c542b..d91a687c 100644 --- a/sandboxd/pool/volume_retirement_test.go +++ b/sandboxd/pool/volume_retirement_test.go @@ -3,6 +3,7 @@ package pool import ( "errors" "os" + "path/filepath" "testing" "github.com/cocoonstack/sandbox/sandboxd/types" @@ -56,3 +57,148 @@ func TestDetachedVolumeRetainsRecoveryProtectionUntilCommit(t *testing.T) { t.Fatalf("writer still blocked after successful retirement: %v", err) } } + +func TestDetachedVolumeOwnRetryAndLaterAcknowledgment(t *testing.T) { + for _, retry := range []string{"empty", "same-shape", "unrelated-snapshot"} { + t.Run(retry, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("detach result unexpectedly persisted") + } + eng.liveVolumeHook = nil + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("physical count retained after detach: %+v", got) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("empty retry acknowledged an unretired file") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mount: "/different"}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("recovery-only shape changed: %v", err) + } + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.States) != 1 || view.States[0].State != types.VolumeDetaching { + t.Fatalf("unretired view: %+v, %v", view, err) + } + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + switch retry { + case "empty": + _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + case "same-shape": + for range 2 { + if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + if got := volumeHoldersOf(m, "data"); got.readers != 1 || got.writers != 0 { + t.Fatalf("own retry counts: %+v", got) + } + } + _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + case "unrelated-snapshot": + _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) + } + if err != nil { + t.Fatal(err) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("retired count: %+v", got) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("writer after acknowledgment: %v", err) + } + }) + } +} + +func TestFailedClaimIntroductionAfterRenameRemainsProtected(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + m.store.recommitting.Store(true) + calls := 0 + m.store.replaceFile = func(raw []byte) error { + calls++ + if calls == 1 { + if err := m.store.replace(raw); err != nil { + return err + } + return errors.New("directory sync failed after rename") + } + return errors.New("replacement failed before rename") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err == nil { + t.Fatal("expected post-rename intent error") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("expected empty snapshot error") + } + if len(sb.Volumes) != 0 || len(sb.VolumesWant) != 0 { + t.Fatalf("memory not empty: %+v", sb) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("unexpected physical count: %+v", got) + } + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil || len(claims[sb.ID].VolumesWant) != 1 { + t.Fatalf("actual recovery file: %+v, %v", claims, err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("writer admitted after uncertain introduction: %v", err) + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "uncertain-introduction", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture admitted: %v", err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err := m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("restart admitted writer: %v", err) + } + m.store.replaceFile = nil + if err := m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + if len(m.store.recoveryVolumes()[sb.ID]) != 0 { + t.Fatal("successful whole snapshot did not retire attempted row") + } +} + +func TestClaimRecoveryReferencesSurviveResetAndAreCopied(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + s = newClaimStore(filepath.Dir(s.path), false) + if _, err := s.load(); err != nil { + t.Fatal(err) + } + s.reset(map[string]*types.Sandbox{}) + refs := s.recoveryVolumes() + refs[sb.ID][0].Name = "caller-copy" + if got := s.recoveryVolumes()[sb.ID]; len(got) != 1 || got[0].Name != "data" { + t.Fatalf("reset or caller erased actual file: %+v", got) + } + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if len(s.recoveryVolumes()) != 0 { + t.Fatal("empty snapshot did not retire file references") + } +} From 3b283cc9518b64fd57dfec6d83055b1de810c1eb Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 13:58:13 +0000 Subject: [PATCH 04/72] sandboxd: test runtime volume registration HTTP contract. --- sandboxd/server/server_test.go | 10 + sandboxd/server/volume_registration_test.go | 234 ++++++++++++++++++++ 2 files changed, 244 insertions(+) create mode 100644 sandboxd/server/volume_registration_test.go diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 6debba66..f24769d1 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2482,6 +2482,8 @@ type checkpointClaimFunc func(string) (*types.Sandbox, error) type idActionFunc func(string) error type fakeManager struct { + registerVolume func(context.Context, string, types.VolumeRegistration) error + unregisterVolume func(context.Context, string) error getVolumes func(string, string) (types.SandboxVolumesResponse, error) setVolumes func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) patchVolumes func(string, string, types.VolumePatch) (types.SandboxVolumesResponse, error) @@ -2557,6 +2559,14 @@ type fakeManager struct { reader *fakeReader } +func (f *fakeManager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { + return f.registerVolume(ctx, name, spec) +} + +func (f *fakeManager) UnregisterVolume(ctx context.Context, name string) error { + return f.unregisterVolume(ctx, name) +} + func (f *fakeManager) GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) { return f.getVolumes(id, tenant) } diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go new file mode 100644 index 00000000..9e9def77 --- /dev/null +++ b/sandboxd/server/volume_registration_test.go @@ -0,0 +1,234 @@ +package server + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestVolumeRegistrationRoutesRequireRoot(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name, apiToken, token string + code int + }{ + {"root", "root", "root", http.StatusNoContent}, + {"tenant", "root", "acme-token", http.StatusForbidden}, + {"sandbox", "root", "sandbox-token", http.StatusUnauthorized}, + {"unknown", "root", "unknown", http.StatusUnauthorized}, + {"missing", "root", "", http.StatusUnauthorized}, + {"open without token", "", "", http.StatusNoContent}, + {"open with tenant token", "", "acme-token", http.StatusNoContent}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + called := false + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, + unregisterVolume: func(context.Context, string) error { called = true; return nil }, + } + ts := newTenantTestServer(t, tt.apiToken, []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) + body := `{"path":"/private/data.img"}` + if tt.code != http.StatusNoContent { + body = `{invalid` + } + resp := doReq(t, method, ts.URL+"/v1/volumes/data", tt.token, body) + defer resp.Body.Close() + got, _ := io.ReadAll(resp.Body) + if resp.StatusCode != tt.code || called != (tt.code == http.StatusNoContent) { + t.Fatalf("status=%d called=%v body=%s, want %d", resp.StatusCode, called, got, tt.code) + } + if tt.code == http.StatusNoContent && len(got) != 0 { + t.Fatalf("204 response body = %q", got) + } + }) + } + } +} + +func TestVolumeRegistrationRoutesForwardBindingAndContext(t *testing.T) { + type requestKey struct{} + const body = `{"path":"/private/images/../data.img","directio":"auto","writable":true,"tenants":["beta","acme"]}` + want := types.VolumeRegistration{Path: "/private/images/../data.img", DirectIO: "auto", Writable: true, Tenants: []string{"beta", "acme"}} + var registered types.VolumeRegistration + var registeredName, removedName string + checkContext := func(ctx context.Context) { + if ctx.Value(requestKey{}) != "request-value" { + t.Error("manager did not receive the request context") + } + } + mgr := &fakeManager{ + registerVolume: func(ctx context.Context, name string, spec types.VolumeRegistration) error { + checkContext(ctx) + registeredName, registered = name, spec + return nil + }, + unregisterVolume: func(ctx context.Context, name string) error { + checkContext(ctx) + removedName = name + return nil + }, + } + handler := New("root", "", mgr, nil, nil, nil, nil, nil).Handler() + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for range 2 { + req := httptest.NewRequest(method, "/v1/volumes/data-v1", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer root") + req = req.WithContext(context.WithValue(req.Context(), requestKey{}, "request-value")) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent || w.Body.Len() != 0 { + t.Fatalf("%s status=%d body=%s, want empty 204", method, w.Code, w.Body) + } + } + } + if registeredName != "data-v1" || removedName != "data-v1" || !reflect.DeepEqual(registered, want) { + t.Fatalf("registered %q %+v, removed %q, want data-v1 %+v", registeredName, registered, removedName, want) + } +} + +func TestVolumeRegistrationRejectsInvalidBodies(t *testing.T) { + for _, tt := range []struct{ name, body string }{ + {"missing", ""}, + {"malformed", `{"path":`}, + {"array", `[]`}, + {"unknown field", `{"path":"/private/data.img","extra":true}`}, + {"body name", `{"name":"other","path":"/private/data.img"}`}, + {"wrong directio key", `{"path":"/private/data.img","direct_io":"on"}`}, + {"duplicate field", `{"path":"/private/a.img","path":"/private/b.img"}`}, + {"trailing object", `{"path":"/private/data.img"}{}`}, + {"trailing text", `{"path":"/private/data.img"}oops`}, + {"wrong writable type", `{"path":"/private/data.img","writable":"true"}`}, + {"wrong tenants type", `{"path":"/private/data.img","tenants":"acme"}`}, + {"oversized", `{"path":"/` + strings.Repeat("a", maxBodyBytes) + `"}`}, + } { + t.Run(tt.name, func(t *testing.T) { + called := false + mgr := &fakeManager{registerVolume: func(context.Context, string, types.VolumeRegistration) error { + called = true + return nil + }} + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, http.MethodPut, ts.URL+"/v1/volumes/data", "root", tt.body) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusBadRequest || called { + t.Fatalf("status=%d called=%v body=%s, want 400 without registration", resp.StatusCode, called, body) + } + }) + } +} + +func TestVolumeRegistrationRoutesValidateNames(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name, encoded string + valid bool + }{ + {"data", "data", true}, + {"a_b-0", "a_b-0", true}, + {"abcdefghijklmnopqrst", "abcdefghijklmnopqrst", true}, + {"Data", "Data", false}, + {"1data", "1data", false}, + {"cocoon-data", "cocoon-data", false}, + {"abcdefghijklmnopqrstu", "abcdefghijklmnopqrstu", false}, + {"data/set", "data%2Fset", false}, + {"data set", "data%20set", false}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + var gotName string + mgr := &fakeManager{ + registerVolume: func(_ context.Context, name string, _ types.VolumeRegistration) error { gotName = name; return nil }, + unregisterVolume: func(_ context.Context, name string) error { gotName = name; return nil }, + } + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, method, ts.URL+"/v1/volumes/"+tt.encoded, "root", `{"path":"/private/data.img"}`) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + wantCode, wantName := http.StatusBadRequest, "" + if tt.valid { + wantCode, wantName = http.StatusNoContent, tt.name + } + if resp.StatusCode != wantCode || gotName != wantName { + t.Fatalf("status=%d name=%q body=%s, want %d name=%q", resp.StatusCode, gotName, body, wantCode, wantName) + } + }) + } + } +} + +func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name string + err error + code int + }{ + {"invalid registration", fmt.Errorf("%w: invalid directio", pool.ErrBadVolume), http.StatusBadRequest}, + {"binding conflict", fmt.Errorf("%w: /private/data.img", pool.ErrVolumeRegistrationConflict), http.StatusConflict}, + {"busy", fmt.Errorf("%w: /private/data.img", pool.ErrVolumeBusy), http.StatusConflict}, + {"catalog unavailable", fmt.Errorf("%w: sync /private/runtime-volumes.json", pool.ErrVolumeCatalogUnavailable), http.StatusServiceUnavailable}, + {"unexpected failure", errors.New("open /private/data.img: permission denied"), http.StatusInternalServerError}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { return tt.err }, + unregisterVolume: func(context.Context, string) error { return tt.err }, + } + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, method, ts.URL+"/v1/volumes/data", "root", `{"path":"/private/data.img"}`) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + var got map[string]string + if err := json.Unmarshal(body, &got); err != nil || len(got) != 1 || got["error"] == "" { + t.Fatalf("error body = %s, decode error = %v", body, err) + } + if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") { + t.Fatalf("status=%d body=%s, want safe %d", resp.StatusCode, body, tt.code) + } + if tt.code == http.StatusServiceUnavailable { + seconds, err := strconv.Atoi(resp.Header.Get("Retry-After")) + if err != nil || seconds < 1 || !strings.Contains(got["error"], "retry") { + t.Fatalf("503 missing retry guidance: headers=%v body=%s", resp.Header, body) + } + } + }) + } + } +} + +func TestVolumeRegistrationKeepsCatalogListingPrivate(t *testing.T) { + for _, token := range []string{"root", "acme-token"} { + t.Run(token, func(t *testing.T) { + mgr := &fakeManager{volumeCatalog: func(tenant string, _ map[string]int) []types.VolumeInfo { + wantTenant := "" + if token == "acme-token" { + wantTenant = "acme" + } + if tenant != wantTenant { + t.Errorf("listing tenant = %q, want %q", tenant, wantTenant) + } + return []types.VolumeInfo{{Name: "data", DefaultMount: "/volumes/data", SizeBytes: 42, Available: true, Nodes: 1, Writable: true}} + }} + ts := newTenantTestServer(t, "root", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) + resp := doReq(t, http.MethodGet, ts.URL+"/v1/volumes", token, "") + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + const want = `{"volumes":[{"name":"data","default_mount":"/volumes/data","size_bytes":42,"available":true,"nodes":1,"writable":true}]}` + if resp.StatusCode != http.StatusOK || string(body) != want { + t.Fatalf("catalog = %d %s, want 200 %s", resp.StatusCode, body, want) + } + }) + } +} From f301f0df8023c9b1a166f62b1dd668fb88830146 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 14:01:11 +0000 Subject: [PATCH 05/72] sandboxd: expose root runtime volume registration routes. --- docs/live-volumes.md | 10 ++++ docs/sandboxd-api.md | 61 ++++++++++++++++++++- sandboxd/server/server.go | 6 ++ sandboxd/server/volume_registration.go | 33 +++++++++++ sandboxd/server/volume_registration_test.go | 1 - 5 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 sandboxd/server/volume_registration.go diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 790e0d21..c4982662 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -9,6 +9,16 @@ file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. A sandbox token alone cannot manage volumes. +The node root token can [register existing backing files](sandboxd-api.md#put-v1volumesname) +without editing static config. Registration persists only node-local catalog +metadata and does not manage disk files. For a sleep and wake workflow, an +external controller owns quiescence and unmount before detach, a reservation +through sleep, and access gating until wake, attach, and mount complete. That +gate also covers business requests that wake the sandbox. Wake establishes +only silkd readiness. The API does not include a controller implementation or +verify that composed workflow. Live changes remain limited to mounted +read-only entries. + Pause application reads before replacement. Close files, mmap mappings, cwd, and bind mount references on the old disk. Replacement unmounts the old disk before mounting the new one, so the path has a short unavailable interval. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 31a255cd..12c0785e 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -8,7 +8,7 @@ All bodies are JSON. Three token kinds: preview mint) and the tenant-scoped listings/deletes below; everything a tenant creates is stamped with its name. Operator surfaces (the per-id sandbox reads, `PUT /v1/sandboxes/{id}/instance-metadata`, `GET /v1/info`, - `PUT /v1/pools`, `POST/DELETE /v1/drain`, `GET /metrics`, + `PUT /v1/pools`, `PUT/DELETE /v1/volumes/{name}`, `POST/DELETE /v1/drain`, `GET /metrics`, `GET /v1/checkpoints/{id}/blob`) answer a tenant token `403` — authenticated but not authorized; an unknown token stays `401`. On a node with a shared @@ -255,6 +255,65 @@ list; the field is emitted (as `true`) only for a writable entry this node declares and omitted otherwise (peer-only rows never carry it), so a read-only entry's response is byte-identical to v1. +## PUT /v1/volumes/{name} + +Auth: root-only node API token. An empty configured `api_token` preserves the +existing open-root mode. A tenant token returns 403 when authentication is +enabled. Sandbox, unknown, and missing tokens return 401. + +Registers an existing backing file on this node. The URL supplies the name; +the body has no `name` field. Unknown fields, duplicate fields, and trailing +JSON return 400. + +```json +{"path": "/srv/datasets/data.img", "directio": "off", + "writable": false, "tenants": ["acme"]} +``` + +- `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. +- `path` uses the same absolute-path validation as static volume config. +- `directio` is `on`, `off`, or `auto`, and defaults to `off`. +- `writable` defaults to `false` and controls writable claim admission. +- `tenants` limits use to the named tenants. Omitted or empty permits all tenants. + +Success is 204 with an empty body. Repeating the same name, cleaned path, and +normalized registration fields succeeds. A different binding or different +registration fields return 409. Static and runtime registrations cannot share +a name or a cleaned backing path. Invalid registration fields return 400. +Unresolved catalog persistence returns 503 with `Retry-After`; retry the request. +GET retains its existing projection without host paths or tenant access lists. +Registration does not expand live mutation beyond mounted read-only volumes. + +Only registration metadata is persisted locally in +`data_dir/runtime-volumes.json`, which loads before claim recovery. The record +is root-owned with mode 0600, and its directory must protect the record. +Runtime mutations require sandboxd to run as OS root. The backing file does +not require root ownership. Registration creates, grows, and deletes no disks. +Catalog replacement and synchronization are independent of `sync_claims`, which +controls claim snapshot synchronization. Process-restart checks do not prove +power-loss recovery. The catalog is node-local with one sandboxd writer. + +## DELETE /v1/volumes/{name} + +Auth and name validation match PUT. Removes only the runtime registration and +never deletes or changes the backing file. Success, including an absent, +unreferenced runtime name, is 204 with an empty body. Static registrations and +names referenced by a claim, an in-flight operation, cleanup, or an +unacknowledged recovery snapshot return 409. Catalog persistence uncertainty +returns 503 with `Retry-After`; retry the request. An empty GET volume view alone +does not establish that a recovery reference has retired. + +An external controller owns quiescence and guest unmount before detach. It +confirms successful detach and recovery bookkeeping before hibernation, and +keeps its own volume reservation while the sandbox sleeps. The controller +gates every access path, including business requests that wake the sandbox, +until wake, attach, and mount complete. Wake establishes only silkd readiness. +It does not restore volume attachments or application readiness. Controller +orchestration is external; no controller implementation or composed lifecycle +verification is included here. Downgrading to a daemon without runtime +registration requires draining runtime references or supplying equivalent +static volume config. Pending live volume changes must finish before downgrade. + ## GET /v1/sandboxes/{id}/volumes Returns the persisted desired set (`volumes_want`), confirmed applied set diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 78b44fd6..70279f63 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -47,6 +47,8 @@ var poolErrHTTP = []struct { }{ {pool.ErrBadKey, http.StatusBadRequest, ""}, {pool.ErrBadVolume, http.StatusBadRequest, ""}, + {pool.ErrVolumeRegistrationConflict, http.StatusConflict, pool.ErrVolumeRegistrationConflict.Error()}, + {pool.ErrVolumeCatalogUnavailable, http.StatusServiceUnavailable, "volume catalog unavailable; retry"}, {pool.ErrBadName, http.StatusBadRequest, ""}, {pool.ErrBadCount, http.StatusBadRequest, ""}, {pool.ErrBadEnv, http.StatusBadRequest, ""}, @@ -97,6 +99,8 @@ type Manager interface { TenantClaims() map[string]int VolumePlacement(key types.PoolKey, tenant string, names []string) (bool, error) Volumes(tenant string, holders map[string]int) []types.VolumeInfo + RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error + UnregisterVolume(ctx context.Context, name string) error GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) SetVolumes(ctx context.Context, id, tenant string, volumes []types.Volume) (types.SandboxVolumesResponse, error) PatchVolumes(ctx context.Context, id, tenant string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) @@ -269,6 +273,8 @@ func (s *Server) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/claim", s.requireToken(s.handleClaim)) mux.HandleFunc("GET /v1/volumes", s.requireToken(s.handleVolumes)) + mux.HandleFunc("PUT /v1/volumes/{name}", s.requireRoot(s.handleVolumeRegistration)) + mux.HandleFunc("DELETE /v1/volumes/{name}", s.requireRoot(s.handleVolumeRegistration)) mux.HandleFunc("POST /v1/sandboxes/{id}/release", s.handleSandboxVerb("release", s.mgr.Release)) mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate)) mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake)) diff --git a/sandboxd/server/volume_registration.go b/sandboxd/server/volume_registration.go new file mode 100644 index 00000000..80de93e4 --- /dev/null +++ b/sandboxd/server/volume_registration.go @@ -0,0 +1,33 @@ +package server + +import ( + "errors" + "net/http" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func (s *Server) handleVolumeRegistration(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if !types.ValidVolumeName(name) { + writeErr(w, http.StatusBadRequest, "invalid volume name") + return + } + var err error + op := "unregister volume" + if r.Method == http.MethodPut { + req, ok := decodeBodyStrict[types.VolumeRegistration](w, r) + if !ok { + return + } + op = "register volume" + err = s.mgr.RegisterVolume(r.Context(), name, req) + } else { + err = s.mgr.UnregisterVolume(r.Context(), name) + } + if errors.Is(err, pool.ErrVolumeBusy) { + err = pool.ErrVolumeBusy + } + writeResult(w, r, op, name, op+" failed", err, noContent(w)) +} diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go index 9e9def77..ba4d4f79 100644 --- a/sandboxd/server/volume_registration_test.go +++ b/sandboxd/server/volume_registration_test.go @@ -105,7 +105,6 @@ func TestVolumeRegistrationRejectsInvalidBodies(t *testing.T) { {"array", `[]`}, {"unknown field", `{"path":"/private/data.img","extra":true}`}, {"body name", `{"name":"other","path":"/private/data.img"}`}, - {"wrong directio key", `{"path":"/private/data.img","direct_io":"on"}`}, {"duplicate field", `{"path":"/private/a.img","path":"/private/b.img"}`}, {"trailing object", `{"path":"/private/data.img"}{}`}, {"trailing text", `{"path":"/private/data.img"}oops`}, From 242b3142841e029af1630bc36f3be2703c69acf7 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 14:03:27 +0000 Subject: [PATCH 06/72] pool: Persist root-managed runtime volume catalog. --- sandboxd/config/config.go | 11 +- sandboxd/pool/claim.go | 6 +- sandboxd/pool/claims.go | 44 ++- sandboxd/pool/pool.go | 25 +- sandboxd/pool/volume.go | 99 +++++-- sandboxd/pool/volume_catalog.go | 289 +++++++++++++++++++ sandboxd/pool/volume_catalog_test.go | 411 +++++++++++++++++++++++++++ sandboxd/pool/volume_mutation.go | 42 ++- 8 files changed, 877 insertions(+), 50 deletions(-) create mode 100644 sandboxd/pool/volume_catalog.go create mode 100644 sandboxd/pool/volume_catalog_test.go diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index fba3efba..d2b2f7dd 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -398,9 +398,14 @@ func (c *Config) validateReplication() error { } func (c *Config) validateVolumes() error { - names := make(map[string]struct{}, len(c.Volumes)) - paths := make(map[string]string, len(c.Volumes)) - for _, volume := range c.Volumes { + return ValidateVolumeCatalog(c.Volumes) +} + +// ValidateVolumeCatalog checks the shared static and runtime catalog rules without I/O. +func ValidateVolumeCatalog(volumes []VolumeSpec) error { + names := make(map[string]struct{}, len(volumes)) + paths := make(map[string]string, len(volumes)) + for _, volume := range volumes { if !types.ValidVolumeName(volume.Name) { return fmt.Errorf("volume name %q must match %s and not start with cocoon-", volume.Name, types.VolumeNameRe) } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 683074ee..3d3be48a 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -371,7 +371,7 @@ func (m *Manager) overQuota(extra int, tenant string) error { // admitVolumes takes one claim's holds; a refusal on either check leaves nothing held. func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types.Volume, error) { applied := appliedVolumes(volumes) - if err := m.admitClaim(tenant, applied); err != nil { + if err := m.admitClaim(tenant, volumes); err != nil { return nil, err } if err := m.confirmVolumesClean(volumes); err != nil { @@ -382,13 +382,13 @@ func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types } // admitClaim refuses a busy volume or an over-quota tenant before any VM is built. -func (m *Manager) admitClaim(tenant string, volumes []types.Volume) error { +func (m *Manager) admitClaim(tenant string, volumes []resolvedVolume) error { m.mu.Lock() defer m.mu.Unlock() if err := m.quotaErr(1, tenant); err != nil { return err } - return m.reserveVolumes(volumes) + return m.reserveResolved(tenant, volumes, nil) } // quotaErr answers ErrQuota when the node takes no more claims; callers hold m.mu. diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 9c3bb874..de8aac12 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -88,7 +88,7 @@ type claimStore struct { seq uint64 written uint64 ackRows []*claimRow - attemptRows []*claimRow + attemptRows map[*claimRow]struct{} replaceFile func([]byte) error recommitting atomic.Bool @@ -118,7 +118,9 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { s.mu.Lock() s.ackRows = make([]*claimRow, 0, len(claims)) for _, sb := range claims { - s.ackRows = append(s.ackRows, newClaimRow(sb)) + if row := newClaimRow(sb); len(row.refs) > 0 { + s.ackRows = append(s.ackRows, row) + } } s.mu.Unlock() return claims, nil @@ -198,9 +200,12 @@ func (s *claimStore) commit(snap claimSnapshot) error { } s.buf = append(s.buf, '}') s.mu.Lock() + if s.attemptRows == nil { + s.attemptRows = map[*claimRow]struct{}{} + } for _, row := range rows { - if !slices.Contains(s.attemptRows, row) { - s.attemptRows = append(s.attemptRows, row) + if len(row.refs) > 0 { + s.attemptRows[row] = struct{}{} } } s.mu.Unlock() @@ -212,7 +217,12 @@ func (s *claimStore) commit(snap claimSnapshot) error { return err } s.mu.Lock() - s.ackRows, s.attemptRows = rows, nil + s.ackRows, s.attemptRows = nil, nil + for _, row := range rows { + if len(row.refs) > 0 { + s.ackRows = append(s.ackRows, row) + } + } s.written = seq s.mu.Unlock() return nil @@ -314,18 +324,28 @@ func newClaimRow(sb *types.Sandbox) *claimRow { } // recoveryVolumes includes every reference a failed replacement may have installed. -func (s *claimStore) recoveryVolumes() map[string][]types.Volume { +func (s *claimStore) recoveryVolumes(owner ...string) map[string][]types.Volume { s.mu.Lock() defer s.mu.Unlock() + if len(s.ackRows) == 0 && len(s.attemptRows) == 0 { + return nil + } refs := map[string][]types.Volume{} - for _, rows := range [][]*claimRow{s.ackRows, s.attemptRows} { - for _, row := range rows { - for _, v := range row.refs { - if !slices.Contains(refs[row.id], v) { - refs[row.id] = append(refs[row.id], v) - } + add := func(row *claimRow) { + if len(owner) > 0 && row.id != owner[0] { + return + } + for _, v := range row.refs { + if !slices.Contains(refs[row.id], v) { + refs[row.id] = append(refs[row.id], v) } } } + for _, row := range s.ackRows { + add(row) + } + for row := range s.attemptRows { + add(row) + } return refs } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 128d4492..5693007a 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -326,12 +326,17 @@ type PeerDeleteFunc func(ctx context.Context, id string) // Manager owns the node's pools, claims, and their persistence. type Manager struct { - eng Engine - dataDir string - egress bool - maxFork int - store *claimStore - volumes map[string]catalogVolume + eng Engine + dataDir string + egress bool + maxFork int + store *claimStore + volumes map[string]*catalogVolume + runtimeVolumes map[string]types.VolumeRegistration + catalogPending map[string]types.VolumeRegistration + catalogWriteMu sync.Mutex + catalogReplace func(string, []byte) error + volumeStat func(string) (os.FileInfo, error) // volumeAdmission counts the live holders of each volume name; guarded by m.mu. volumeAdmission map[string]volumeHolders volumeRuns map[*types.Sandbox]*volumeRun @@ -453,7 +458,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg cocoondSocket: cfg.CocoondSocket, maxFork: cmp.Or(cfg.MaxForkCount, defaultMaxFork), store: newClaimStore(cfg.DataDir, cfg.SyncClaims), - volumes: make(map[string]catalogVolume, len(cfg.Volumes)), + volumes: make(map[string]*catalogVolume, len(cfg.Volumes)), volumeAdmission: map[string]volumeHolders{}, volumeRuns: map[*types.Sandbox]*volumeRun{}, volumeErrors: map[*types.Sandbox]map[string]string{}, @@ -483,12 +488,15 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg healSem: make(chan struct{}, maxConcurrentHeals), } for _, volume := range cfg.Volumes { - m.volumes[volume.Name] = catalogVolume{ + m.volumes[volume.Name] = &catalogVolume{ disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: volume.DirectIO}, tenants: slices.Clone(volume.Tenants), writable: volume.Writable, } } + if err := m.loadRuntimeVolumes(); err != nil { + return nil, err + } if err := os.MkdirAll(m.goldensDir(), 0o750); err != nil { return nil, fmt.Errorf("create goldens dir: %w", err) } @@ -605,6 +613,7 @@ func (m *Manager) Run(ctx context.Context) { case <-m.refillKick: m.refillOnce(ctx) case <-reap.C: + go m.retryVolumeCatalog(ctx) m.retryRemovals(ctx) m.reapOnce(ctx) m.idleOnce(ctx) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 12810904..b880d297 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "maps" "os" "path/filepath" "slices" @@ -31,6 +32,7 @@ type volumeHolders struct { } type resolvedVolume struct { + entry *catalogVolume disk engine.VolumeSpec applied types.Volume } @@ -53,8 +55,9 @@ func (v catalogVolume) allowed(tenant string) bool { // Volumes reports the caller-visible fleet catalog; an empty tenant means root. func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeInfo { - infos := make([]types.VolumeInfo, 0, len(m.volumes)) - for name, volume := range m.volumes { + catalog := m.volumeCatalog() + infos := make([]types.VolumeInfo, 0, len(catalog)) + for name, volume := range catalog { if !volume.allowed(tenant) { continue } @@ -64,7 +67,7 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI Nodes: holders[name], Writable: volume.writable, } - if st, err := os.Stat(volume.disk.Path); err == nil { + if st, err := m.statVolume(volume.disk.Path); err == nil { info.SizeBytes = st.Size() info.Available = true info.Nodes = max(info.Nodes, 1) @@ -74,7 +77,7 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI // root can see peer-only names; a tenant needs local catalog metadata to apply the access list if tenant == "" { for name, nodes := range holders { - if _, ok := m.volumes[name]; !ok { + if _, ok := catalog[name]; !ok { infos = append(infos, types.VolumeInfo{ Name: name, DefaultMount: types.DefaultVolumeMount(name), @@ -89,9 +92,10 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI // VolumeNames returns the sorted names this node can currently serve. func (m *Manager) VolumeNames() []string { - names := make([]string, 0, len(m.volumes)) - for name, volume := range m.volumes { - if _, err := os.Stat(volume.disk.Path); err == nil { + catalog := m.volumeCatalog() + names := make([]string, 0, len(catalog)) + for name, volume := range catalog { + if _, err := m.statVolume(volume.disk.Path); err == nil { names = append(names, name) } } @@ -108,8 +112,9 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri return true, nil } local := true + catalog := m.volumeCatalog() for _, name := range names { - volume, ok := m.volumes[name] + volume, ok := catalog[name] if !ok { if tenant == "" { local = false @@ -120,7 +125,7 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri if !volume.allowed(tenant) { return false, ErrVolumeUnavailable } - if _, err := os.Stat(volume.disk.Path); err != nil { + if _, err := m.statVolume(volume.disk.Path); err != nil { local = false } } @@ -136,13 +141,20 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } + m.mu.Lock() + if m.catalogPending != nil { + m.mu.Unlock() + return nil, ErrVolumeCatalogUnavailable + } + catalog := maps.Clone(m.volumes) + m.mu.Unlock() resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { - entry, ok := m.volumes[volume.Name] + entry, ok := catalog[volume.Name] if !ok || !entry.allowed(tenant) { return nil, ErrVolumeUnavailable } - resolvedEntry, resolveErr := m.resolveCatalogVolume(ctx, volume) + resolvedEntry, resolveErr := m.resolveCatalogEntry(ctx, volume, entry) if resolveErr != nil { return nil, resolveErr } @@ -152,14 +164,24 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ } func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) (resolvedVolume, error) { - entry, ok := m.volumes[volume.Name] - if !ok { + m.mu.Lock() + entry := m.volumes[volume.Name] + pending := m.catalogPending != nil + m.mu.Unlock() + if pending && entry == nil { + return resolvedVolume{}, ErrVolumeCatalogUnavailable + } + return m.resolveCatalogEntry(ctx, volume, entry) +} + +func (m *Manager) resolveCatalogEntry(ctx context.Context, volume types.Volume, entry *catalogVolume) (resolvedVolume, error) { + if entry == nil { return resolvedVolume{}, ErrVolumeUnavailable } if volume.RW() && !entry.writable { return resolvedVolume{}, fmt.Errorf("%w: volume %q is not writable", ErrBadVolume, volume.Name) } - if _, statErr := os.Stat(entry.disk.Path); statErr != nil { + if _, statErr := m.statVolume(entry.disk.Path); statErr != nil { // the operator's path stays server-side; the refusal matches the unknown one log.WithFunc("pool.resolveCatalogVolume").Warnf(ctx, "stat volume %q image %s: %v", volume.Name, entry.disk.Path, statErr) return resolvedVolume{}, ErrVolumeUnavailable @@ -167,7 +189,7 @@ func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) disk := entry.disk disk.RW = volume.RW() volume.AttachOnly = false // past validation the empty mount carries it - return resolvedVolume{disk: disk, applied: volume}, nil + return resolvedVolume{entry: entry, disk: disk, applied: volume}, nil } // confirmVolumesClean refuses a read-only claim of a dirty image; run after holds are taken. @@ -246,9 +268,11 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT stuck = true continue } + m.mu.Lock() if entry, ok := m.volumes[volume.Name]; ok { td.clears = append(td.clears, entry.disk.Path) } + m.mu.Unlock() } if stuck { // the unmounts may have spent the whole budget; the fallback flush gets its own @@ -281,11 +305,17 @@ func (m *Manager) reserveVolumes(volumes []types.Volume) error { } func (m *Manager) reserveVolumesFor(volumes []types.Volume, owner *types.Sandbox) error { + if len(volumes) == 0 { + return nil + } + if len(volumes) > 0 && m.catalogPending != nil { + return ErrVolumeCatalogUnavailable + } recovery := m.store.recoveryVolumes() for _, volume := range volumes { for id, refs := range recovery { for _, ref := range refs { - if owner != nil && m.claimed[id] == owner && owner.ID == id && ref == volume { + if owner != nil && m.claimed[id] == owner && m.volumeRuns[owner] != nil && owner.ID == id && ref == volume { continue } if ref.Name == volume.Name && (ref.RW() || volume.RW()) { @@ -404,3 +434,40 @@ func syncDir(dir string) error { } return errors.Join(d.Sync(), d.Close()) } + +func (m *Manager) volumeCatalog() map[string]*catalogVolume { + m.mu.Lock() + defer m.mu.Unlock() + return maps.Clone(m.volumes) +} + +// reserveResolved revalidates the binding and policy in the same decision as admission. +func (m *Manager) reserveResolved(tenant string, volumes []resolvedVolume, owner *types.Sandbox) error { + if len(volumes) > 0 && m.catalogPending != nil { + return ErrVolumeCatalogUnavailable + } + var held []types.Volume + if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { + held = slices.Concat(heldVolumes(owner), m.store.recoveryVolumes(owner.ID)[owner.ID]) + } + for _, v := range volumes { + entry := m.volumes[v.applied.Name] + if entry == nil || entry != v.entry { + return ErrVolumeUnavailable + } + if !slices.Contains(held, v.applied) && !entry.allowed(tenant) { + return ErrVolumeUnavailable + } + if v.applied.RW() && !entry.writable { + return fmt.Errorf("%w: volume %q is not writable", ErrBadVolume, v.applied.Name) + } + } + return m.reserveVolumesFor(appliedVolumes(volumes), owner) +} + +func (m *Manager) statVolume(path string) (os.FileInfo, error) { + if m.volumeStat != nil { + return m.volumeStat(path) + } + return os.Stat(path) +} diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go new file mode 100644 index 00000000..1b6cc928 --- /dev/null +++ b/sandboxd/pool/volume_catalog.go @@ -0,0 +1,289 @@ +package pool + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "syscall" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +const runtimeVolumesFile = "runtime-volumes.json" + +type runtimeCatalog struct { + Version int `json:"version"` + Volumes map[string]types.VolumeRegistration `json:"volumes"` +} + +func normalizeRegistration(name string, spec types.VolumeRegistration) (types.VolumeRegistration, error) { + if spec.DirectIO == "" { + spec.DirectIO = types.DirectIOOff + } + if err := config.ValidateVolumeCatalog([]config.VolumeSpec{{Name: name, Path: spec.Path, DirectIO: spec.DirectIO}}); err != nil { + return spec, fmt.Errorf("%w: %w", ErrBadVolume, err) + } + spec.Path = filepath.Clean(spec.Path) + spec.Tenants = slices.Clone(spec.Tenants) + slices.Sort(spec.Tenants) + spec.Tenants = slices.Compact(spec.Tenants) + return spec, nil +} + +func sameRegistration(a, b types.VolumeRegistration) bool { + return a.Path == b.Path && a.DirectIO == b.DirectIO && a.Writable == b.Writable && slices.Equal(a.Tenants, b.Tenants) +} + +func (m *Manager) loadRuntimeVolumes() error { + m.runtimeVolumes = map[string]types.VolumeRegistration{} + path := filepath.Join(m.dataDir, runtimeVolumesFile) + st, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("inspect runtime volume catalog: %w", err) + } + if err = protectedCatalogPath(m.dataDir); err != nil { + return err + } + if !st.Mode().IsRegular() || st.Mode().Perm() != 0o600 || st.Sys().(*syscall.Stat_t).Uid != 0 { + return fmt.Errorf("runtime volume catalog must be a root-owned regular file with mode 0600") + } + raw, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read runtime volume catalog: %w", err) + } + var record runtimeCatalog + if err = json.Unmarshal(raw, &record, json.RejectUnknownMembers(true)); err != nil { + return fmt.Errorf("parse runtime volume catalog: %w", err) + } + if record.Version != 1 || record.Volumes == nil { + return fmt.Errorf("invalid runtime volume catalog version or volumes") + } + merged := make([]config.VolumeSpec, 0, len(m.volumes)+len(record.Volumes)) + for name, entry := range m.volumes { + merged = append(merged, config.VolumeSpec{Name: name, Path: entry.disk.Path, DirectIO: entry.disk.DirectIO}) + } + for name, spec := range record.Volumes { + normalized, err := normalizeRegistration(name, spec) + if err != nil { + return fmt.Errorf("runtime volume catalog: %w", err) + } + m.runtimeVolumes[name] = normalized + merged = append(merged, config.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}) + } + if err := config.ValidateVolumeCatalog(merged); err != nil { + return fmt.Errorf("runtime volume catalog: %w", err) + } + m.publishRuntimeCatalog(m.runtimeVolumes) + return nil +} + +// RegisterVolume durably binds a root-managed name to an existing backing file. +func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { + if os.Geteuid() != 0 { + return fmt.Errorf("%w: runtime catalog mutation requires effective uid zero", ErrVolumeCatalogUnavailable) + } + m.catalogWriteMu.Lock() + defer m.catalogWriteMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + return err + } + normalized, err := normalizeRegistration(name, spec) + if err != nil { + return err + } + m.mu.Lock() + if current, ok := m.runtimeVolumes[name]; ok { + m.mu.Unlock() + if sameRegistration(current, normalized) { + return nil + } + return fmt.Errorf("%w: volume %q already has different semantics", ErrVolumeRegistrationConflict, name) + } + if m.volumeReferenced(name) { + m.mu.Unlock() + return fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) + } + for existing, entry := range m.volumes { + if existing == name || filepath.Clean(entry.disk.Path) == normalized.Path { + m.mu.Unlock() + return fmt.Errorf("%w: volume %q duplicates a catalog name or path", ErrVolumeRegistrationConflict, name) + } + } + m.mu.Unlock() + if _, err := m.statVolume(normalized.Path); err != nil { + return fmt.Errorf("%w: backing for volume %q is unavailable", ErrBadVolume, name) + } + if err := ctx.Err(); err != nil { + return err + } + m.mu.Lock() + // A claim can acquire an unexplained recovery name while the backing is checked. + if m.volumeReferenced(name) { + m.mu.Unlock() + return fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) + } + proposal := maps.Clone(m.runtimeVolumes) + proposal[name] = normalized + m.catalogPending = proposal + m.mu.Unlock() + return m.commitPendingCatalog(ctx) +} + +// UnregisterVolume retires an unreferenced runtime binding without touching its backing. +func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { + if os.Geteuid() != 0 { + return fmt.Errorf("%w: runtime catalog mutation requires effective uid zero", ErrVolumeCatalogUnavailable) + } + m.catalogWriteMu.Lock() + defer m.catalogWriteMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + return err + } + if !types.ValidVolumeName(name) { + return fmt.Errorf("%w: invalid volume name", ErrBadVolume) + } + m.mu.Lock() + if _, runtime := m.runtimeVolumes[name]; !runtime { + _, exists := m.volumes[name] + referenced := m.volumeReferenced(name) + m.mu.Unlock() + if exists || referenced { + return fmt.Errorf("%w: volume %q is static or referenced", ErrVolumeRegistrationConflict, name) + } + return nil + } + if m.volumeReferenced(name) { + m.mu.Unlock() + return fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) + } + proposal := maps.Clone(m.runtimeVolumes) + delete(proposal, name) + m.catalogPending = proposal + m.mu.Unlock() + return m.commitPendingCatalog(ctx) +} + +// volumeReferenced includes dispatched cleanup through its ordinary admission counts. +func (m *Manager) volumeReferenced(name string) bool { + if holders := m.volumeAdmission[name]; holders.readers > 0 || holders.writers > 0 { + return true + } + for _, run := range m.volumeRuns { + if volumeIndex(run.holds, name) >= 0 { + return true + } + } + for _, sb := range m.claimed { + if volumeIndex(heldVolumes(sb), name) >= 0 { + return true + } + } + for _, refs := range m.store.recoveryVolumes() { + if volumeIndex(refs, name) >= 0 { + return true + } + } + return false +} + +// commitPendingCatalog holds catalogWriteMu, but never m.mu across filesystem work. +func (m *Manager) commitPendingCatalog(ctx context.Context) error { + m.mu.Lock() + proposal := m.catalogPending + m.mu.Unlock() + if proposal == nil { + return nil + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) + } + if err := m.replaceRuntimeCatalog(proposal); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) + } + m.mu.Lock() + m.publishRuntimeCatalog(proposal) + m.catalogPending = nil + m.mu.Unlock() + return nil +} + +func (m *Manager) publishRuntimeCatalog(proposal map[string]types.VolumeRegistration) { + for name := range m.runtimeVolumes { + if _, ok := proposal[name]; !ok { + delete(m.volumes, name) + } + } + for name, spec := range proposal { + if prior, ok := m.runtimeVolumes[name]; ok && sameRegistration(prior, spec) && m.volumes[name] != nil { + continue + } + m.volumes[name] = &catalogVolume{disk: engine.VolumeSpec{Name: name, Path: spec.Path, DirectIO: spec.DirectIO}, tenants: slices.Clone(spec.Tenants), writable: spec.Writable} + } + m.runtimeVolumes = proposal +} + +func (m *Manager) replaceRuntimeCatalog(proposal map[string]types.VolumeRegistration) error { + if err := protectedCatalogPath(m.dataDir); err != nil { + return err + } + path := filepath.Join(m.dataDir, runtimeVolumesFile) + if st, err := os.Lstat(path); err == nil { + if !st.Mode().IsRegular() || st.Mode().Perm() != 0o600 || st.Sys().(*syscall.Stat_t).Uid != 0 { + return fmt.Errorf("runtime volume catalog must be a root-owned regular file with mode 0600") + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + raw, err := json.Marshal(runtimeCatalog{Version: 1, Volumes: proposal}, json.Deterministic(true)) + if err != nil { + return err + } + if m.catalogReplace != nil { + return m.catalogReplace(path, raw) + } + return utils.WriteFileSync(path, raw, 0o600) +} + +func protectedCatalogPath(dir string) error { + absolute, err := filepath.Abs(dir) + if err != nil { + return err + } + for { + st, err := os.Lstat(absolute) + if err != nil { + return fmt.Errorf("inspect runtime catalog directory: %w", err) + } + if !st.IsDir() || st.Sys().(*syscall.Stat_t).Uid != 0 || st.Mode().Perm()&0o022 != 0 && st.Mode()&os.ModeSticky == 0 { + return fmt.Errorf("runtime catalog directory %q must be protected and root-owned", absolute) + } + parent := filepath.Dir(absolute) + if parent == absolute { + return nil + } + absolute = parent + } +} + +func (m *Manager) retryVolumeCatalog(ctx context.Context) { + if !m.catalogWriteMu.TryLock() { + return + } + defer m.catalogWriteMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + log.WithFunc("pool.retryVolumeCatalog").Error(ctx, err, "persist runtime volume catalog") + } +} diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go new file mode 100644 index 00000000..4e4d489b --- /dev/null +++ b/sandboxd/pool/volume_catalog_test.go @@ -0,0 +1,411 @@ +package pool + +import ( + "errors" + "fmt" + "github.com/cocoonstack/sandbox/sandboxd/config" + "os" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +func TestRuntimeCatalogPersistsNormalizedSemantics(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "backing bytes stay intact") + spec := types.VolumeRegistration{Path: filepath.Dir(path) + "/./runtime", Writable: true, Tenants: []string{"beta", "acme", "acme"}} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + spec.Path = path + spec.DirectIO = types.DirectIOOff + spec.Tenants = []string{"acme", "beta"} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("normalized retry: %v", err) + } + st, err := os.Stat(filepath.Join(m.dataDir, "runtime-volumes.json")) + if err != nil || st.Mode().Perm() != 0o600 { + t.Fatalf("metadata mode: %v %v", st, err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + if got := m2.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("recovered names: %v", got) + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "forbidden", Volumes: []types.Volume{{Name: "runtime"}}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("ACL changed at restart: %v", err) + } + resolved, err := m2.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "runtime", Mode: types.VolumeModeRW}}) + if err != nil || len(resolved) != 1 || resolved[0].disk.DirectIO != "off" || !resolved[0].disk.RW { + t.Fatalf("policy roundtrip: %+v %v", resolved, err) + } + changed := spec + changed.DirectIO = "on" + if err := m2.RegisterVolume(t.Context(), "runtime", changed); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("changed semantics: %v", err) + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatalf("absent retry: %v", err) + } + if got, err := os.ReadFile(path); err != nil || string(got) != "backing bytes stay intact" { + t.Fatalf("backing changed: %q %v", got, err) + } + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("unregister not durable: %v", got) + } +} + +func TestRuntimeCatalogCollisionsAndRecoveryNames(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + static := liveCatalog(t) + m := newVolumeManager(t, eng, static) + other := writeVolumeImage(t, "other-runtime", "bytes") + for _, tc := range []struct{ name, path string }{{"data", other}, {"runtime", static[0].Path}, {"runtime", filepath.Dir(static[0].Path) + "/./" + filepath.Base(static[0].Path)}} { + if err := m.RegisterVolume(t.Context(), tc.name, types.VolumeRegistration{Path: tc.path}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("collision %+v: %v", tc, err) + } + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("static removal: %v", err) + } + sb := mustClaim(t, m, testKey) + sb.Volumes = []types.Volume{{Name: "unexplained", Mount: "/data"}} + if err := m.store.commit(m.store.set(sb)); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "unexplained", types.VolumeRegistration{Path: other}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("rebound unexplained recovery name: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "unexplained"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("absent referenced removal: %v", err) + } + for _, spec := range []types.VolumeRegistration{{Path: "relative"}, {Path: other, DirectIO: "invalid"}, {Path: other + "missing"}} { + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrBadVolume) { + t.Fatalf("invalid spec %+v: %v", spec, err) + } + } +} + +func TestRuntimeCatalogPostRenameFailureFencesAndConverges(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "bytes") + m.catalogReplace = func(path string, raw []byte) error { + if err := utils.WriteFileSync(path, raw, 0o600); err != nil { + return err + } + return errors.New("directory sync failed after rename") + } + spec := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("post-rename failure: %v", err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + if !slices.Equal(m2.VolumeNames(), []string{"runtime"}) { + t.Fatal("actual renamed record not recovered") + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "runtime"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("uncertain admission: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatalf("volume-free claim blocked: %v", err) + } + m.catalogReplace = nil + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("retry pending register: %v", err) + } + m.catalogReplace = func(path string, raw []byte) error { + if err := utils.WriteFileSync(path, raw, 0o600); err != nil { + return err + } + return errors.New("directory sync failed after rename") + } + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("uncertain unregister: %v", err) + } + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("unregister rename did not happen: %v", got) + } + if err := m.RegisterVolume(t.Context(), "replacement", types.VolumeRegistration{Path: path}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("pending unregister reuse: %v", err) + } + m.catalogReplace = nil + if err := m.RegisterVolume(t.Context(), "replacement", types.VolumeRegistration{Path: path}); err != nil { + t.Fatalf("retry did not converge before new request: %v", err) + } +} + +func TestResolvedCatalogBindingCannotSurviveUnregisterReuse(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), nil) + path := writeVolumeImage(t, "runtime", "original") + spec := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) + if err != nil { + t.Fatal(err) + } + if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + spec.Path = writeVolumeImage(t, "replacement", "replacement") + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("stale binding admitted: %v", err) + } + if got := volumeHoldersOf(m, "runtime"); got != (volumeHolders{}) { + t.Fatalf("stale binding leaked reservation: %+v", got) + } +} + +func TestRuntimeCatalogMissingBackingAndUnrestrictedBackingOwner(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if err := os.Chown(path, 65534, 65534); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatalf("nonroot backing owner: %v", err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + infos := m2.Volumes("", nil) + if len(infos) != 1 || infos[0].Available { + t.Fatalf("missing backing lost mapping: %+v", infos) + } + if _, err := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("missing backing resolved: %v", err) + } + if err := m2.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatalf("idempotent missing mapping: %v", err) + } +} + +func requireCatalogRoot(t *testing.T) { + t.Helper() + if os.Geteuid() != 0 { + t.Skip("real runtime metadata mutation requires effective uid zero") + } +} + +func TestRuntimeRegistrationNormalization(t *testing.T) { + input := types.VolumeRegistration{Path: "/images/./data", Tenants: []string{"beta", "acme", "acme"}} + normalized, err := normalizeRegistration("data", input) + if err != nil || normalized.Path != "/images/data" || normalized.DirectIO != types.DirectIOOff || !slices.Equal(normalized.Tenants, []string{"acme", "beta"}) { + t.Fatalf("normalization: %+v %v", normalized, err) + } + normalized.Tenants[0] = "caller-copy" + if input.Tenants[1] != "acme" { + t.Fatal("normalization mutated caller input") + } + for _, tc := range []struct { + name string + spec types.VolumeRegistration + }{{"cocoon-reserved", input}, {"bad/name", input}, {"data", types.VolumeRegistration{Path: "relative"}}, {"data", types.VolumeRegistration{Path: "/data", DirectIO: "bad"}}} { + if _, err := normalizeRegistration(tc.name, tc.spec); !errors.Is(err, ErrBadVolume) { + t.Fatalf("invalid registration %+v: %v", tc, err) + } + } +} + +func TestStaticOnlyCatalogDoesNotNeedRoot(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + if len(m.VolumeNames()) == 0 { + t.Fatal("static-only startup lost catalog") + } + if os.Geteuid() != 0 { + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/data"}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("nonroot write: %v", err) + } + } +} + +func TestRuntimeCatalogRejectsInvalidMetadata(t *testing.T) { + requireCatalogRoot(t) + for _, kind := range []string{"malformed", "version", "null", "name", "relative", "directio", "duplicate-name", "duplicate-path", "static-name", "static-path", "unknown-field", "symlink", "mode", "owner", "directory"} { + t.Run(kind, func(t *testing.T) { + eng := newFakeEngine() + static := liveCatalog(t) + m := newVolumeManager(t, eng, static) + path := writeVolumeImage(t, "runtime", "bytes") + spec := fmt.Sprintf(`{"path":%q,"directio":"auto"}`, path) + raw := fmt.Sprintf(`{"version":1,"volumes":{"runtime":%s}}`, spec) + switch kind { + case "malformed": + raw = "{" + case "version": + raw = `{"version":2,"volumes":{}}` + case "null": + raw = `{"version":1,"volumes":null}` + case "name": + raw = fmt.Sprintf(`{"version":1,"volumes":{"bad/name":%s}}`, spec) + case "relative": + raw = `{"version":1,"volumes":{"runtime":{"path":"relative"}}}` + case "directio": + raw = `{"version":1,"volumes":{"runtime":{"path":"/data","directio":"bad"}}}` + case "duplicate-name": + raw = fmt.Sprintf(`{"version":1,"volumes":{"runtime":%s,"runtime":%s}}`, spec, spec) + case "duplicate-path": + raw = fmt.Sprintf(`{"version":1,"volumes":{"runtime":%s,"second":%s}}`, spec, spec) + case "static-name": + raw = fmt.Sprintf(`{"version":1,"volumes":{"data":%s}}`, spec) + case "static-path": + raw = fmt.Sprintf(`{"version":1,"volumes":{"runtime":{"path":%q}}}`, static[0].Path) + case "unknown-field": + raw = `{"version":1,"volumes":{},"unexpected":true}` + } + record := filepath.Join(m.dataDir, runtimeVolumesFile) + if err := os.WriteFile(record, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + switch kind { + case "symlink": + if err := os.Rename(record, record+".target"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(record+".target", record); err != nil { + t.Fatal(err) + } + case "mode": + if err := os.Chmod(record, 0o644); err != nil { + t.Fatal(err) + } + case "owner": + if err := os.Chown(record, 65534, 65534); err != nil { + t.Fatal(err) + } + case "directory": + if err := os.Chmod(m.dataDir, 0o777); err != nil { + t.Fatal(err) + } + } + cfg := &config.Config{DataDir: m.dataDir, Volumes: static} + if _, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)); err == nil { + t.Fatalf("accepted %s metadata", kind) + } + }) + } +} + +func TestRuntimeCatalogRetirementBlocksUnregisterAndReuse(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "unchanged") + spec := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "data", spec); err != nil { + t.Fatal(err) + } + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("expected failed result commit") + } + eng.liveVolumeHook = nil + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister unretired name: %v", err) + } + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "different", "different")}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("rebind unretired name: %v", err) + } + if err := m.RegisterVolume(t.Context(), "same-file", spec); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("alias unretired path: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("same-file writer: %v", err) + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retirement-fenced", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture: %v", err) + } + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "same-file", spec); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeCatalogIOLeavesManagerAndTransitionUnlocked(t *testing.T) { + requireCatalogRoot(t) + for _, operation := range []string{"stat", "replace"} { + t.Run(operation, func(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + sb := mustClaim(t, m, testKey) + entered, resume := make(chan struct{}), make(chan struct{}) + var once sync.Once + barrier := func() { once.Do(func() { close(entered) }); <-resume } + if operation == "stat" { + m.volumeStat = func(path string) (os.FileInfo, error) { barrier(); return os.Stat(path) } + } else { + m.catalogReplace = func(path string, raw []byte) error { + if err := utils.WriteFileSync(path, raw, 0o600); err != nil { + return err + } + barrier() + return nil + } + } + done := make(chan error, 1) + go func() { + if operation == "stat" { + _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "data"}}) + done <- err + } else { + done <- m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}) + } + }() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("I/O did not block") + } + if !m.mu.TryLock() { + t.Fatal("I/O holds manager lock") + } + m.mu.Unlock() + if !sb.Transition.TryLock() { + t.Fatal("I/O holds transition lock") + } + sb.Transition.Unlock() + close(resume) + if err := <-done; err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index bd9a70ad..e9be5c15 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -86,7 +86,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v }() m.mu.Lock() current, held := slices.Clone(sb.VolumesWant), heldVolumes(sb) - recovery := m.store.recoveryVolumes()[sb.ID] + recovery := m.store.recoveryVolumes(sb.ID)[sb.ID] m.mu.Unlock() target, err := change(current) if err != nil { @@ -105,7 +105,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v return view, err } } - resolved, err := m.resolveVolumes(ctx, sb.Tenant, additions) + resolved, err := m.resolveLiveAdditions(ctx, sb, additions, recovery) if err != nil { return view, err } @@ -258,23 +258,28 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) (claimSnapshot, bool, error) { sb.Transition.Lock() - defer sb.Transition.Unlock() m.mu.Lock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { m.mu.Unlock() + sb.Transition.Unlock() return claimSnapshot{}, false, err } if sb.OnExpire == types.ExpireArchive && len(target) > 0 { m.mu.Unlock() + sb.Transition.Unlock() return claimSnapshot{}, false, ErrVolumeCapture } - if err := m.reserveVolumesFor(additions, sb); err != nil { + if err := m.reserveResolved(sb.Tenant, resolved, sb); err != nil { m.mu.Unlock() + sb.Transition.Unlock() return claimSnapshot{}, false, err } run.holds = append(run.holds, additions...) m.mu.Unlock() + sb.Transition.Unlock() cleanErr := m.confirmVolumesClean(resolved) + sb.Transition.Lock() + defer sb.Transition.Unlock() m.mu.Lock() defer m.mu.Unlock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { @@ -346,18 +351,19 @@ func (m *Manager) ownsRunLocked(sb *types.Sandbox, run *volumeRun) bool { func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { run := m.volumeRuns[sb] - return hasVolumes(sb) || run != nil && len(run.holds) > 0 || len(m.store.recoveryVolumes()[sb.ID]) > 0 + return hasVolumes(sb) || run != nil && len(run.holds) > 0 || len(m.store.recoveryVolumes(sb.ID)[sb.ID]) > 0 } func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { + held := heldVolumes(sb) + recovering := volumeDifference(m.store.recoveryVolumes(sb.ID)[sb.ID], held) view := types.SandboxVolumesResponse{ Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), States: []types.VolumeState{}, - Converging: m.volumeRuns[sb] != nil, + Converging: m.volumeRuns[sb] != nil || len(recovering) > 0, } - held := heldVolumes(sb) - held = append(held, volumeDifference(m.store.recoveryVolumes()[sb.ID], held)...) + held = append(held, recovering...) for _, v := range held { state := types.VolumeAttached if volumeIndex(sb.VolumesWant, v.Name) < 0 { @@ -485,3 +491,23 @@ func validateDetachNames(names []string) error { func heldVolumes(sb *types.Sandbox) []types.Volume { return slices.Concat(sb.Volumes, volumeDifference(sb.VolumesWant, sb.Volumes)) } + +func (m *Manager) resolveLiveAdditions(ctx context.Context, sb *types.Sandbox, additions, recovery []types.Volume) ([]resolvedVolume, error) { + resolved := make([]resolvedVolume, 0, len(additions)) + for _, v := range additions { + if slices.Contains(recovery, v) { + entry, err := m.resolveCatalogVolume(ctx, v) + if err != nil { + return nil, err + } + resolved = append(resolved, entry) + continue + } + entries, err := m.resolveVolumes(ctx, sb.Tenant, []types.Volume{v}) + if err != nil { + return nil, err + } + resolved = append(resolved, entries...) + } + return resolved, nil +} From 7d0a912085c8a1d4a312409912ebc7ede207a8b6 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 14:17:04 +0000 Subject: [PATCH 07/72] pool: Retain volume references through cleanup and stale VM recovery. --- sandboxd/pool/claims.go | 75 +++--- sandboxd/pool/pool.go | 1 + sandboxd/pool/reconcile.go | 44 ++-- sandboxd/pool/remove.go | 26 ++- sandboxd/pool/volume.go | 33 ++- sandboxd/pool/volume_catalog.go | 18 +- sandboxd/pool/volume_catalog_test.go | 11 +- sandboxd/pool/volume_cleanup_test.go | 290 ++++++++++++++++++++++++ sandboxd/pool/volume_mutation.go | 2 +- sandboxd/pool/volume_mutation_test.go | 8 +- sandboxd/pool/volume_retirement_test.go | 75 +++++- 11 files changed, 506 insertions(+), 77 deletions(-) create mode 100644 sandboxd/pool/volume_cleanup_test.go diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index de8aac12..97ebe987 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -87,8 +87,8 @@ type claimStore struct { rows map[string]*claimRow seq uint64 written uint64 - ackRows []*claimRow - attemptRows map[*claimRow]struct{} + ackRefs map[string][]types.Volume + attemptRefs map[string][]types.Volume replaceFile func([]byte) error recommitting atomic.Bool @@ -116,10 +116,13 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { } } s.mu.Lock() - s.ackRows = make([]*claimRow, 0, len(claims)) + s.ackRefs = nil for _, sb := range claims { if row := newClaimRow(sb); len(row.refs) > 0 { - s.ackRows = append(s.ackRows, row) + if s.ackRefs == nil { + s.ackRefs = map[string][]types.Volume{} + } + s.ackRefs[row.id] = row.refs } } s.mu.Unlock() @@ -185,7 +188,14 @@ func (s *claimStore) commit(snap claimSnapshot) error { rows, seq := slices.AppendSeq(make([]*claimRow, 0, len(s.rows)), maps.Values(s.rows)), s.seq s.mu.Unlock() s.buf = append(s.buf[:0], '{') + var refs map[string][]types.Volume for i, row := range rows { + if len(row.refs) > 0 { + if refs == nil { + refs = map[string][]types.Volume{} + } + refs[row.id] = row.refs + } if row.raw == nil { raw, err := encodeRow(&row.dto) if err != nil { @@ -200,14 +210,7 @@ func (s *claimStore) commit(snap claimSnapshot) error { } s.buf = append(s.buf, '}') s.mu.Lock() - if s.attemptRows == nil { - s.attemptRows = map[*claimRow]struct{}{} - } - for _, row := range rows { - if len(row.refs) > 0 { - s.attemptRows[row] = struct{}{} - } - } + s.attemptRefs = mergeRecoveryRefs(s.attemptRefs, refs) s.mu.Unlock() replace := s.replace if s.replaceFile != nil { @@ -217,12 +220,7 @@ func (s *claimStore) commit(snap claimSnapshot) error { return err } s.mu.Lock() - s.ackRows, s.attemptRows = nil, nil - for _, row := range rows { - if len(row.refs) > 0 { - s.ackRows = append(s.ackRows, row) - } - } + s.ackRefs, s.attemptRefs = refs, nil s.written = seq s.mu.Unlock() return nil @@ -327,25 +325,32 @@ func newClaimRow(sb *types.Sandbox) *claimRow { func (s *claimStore) recoveryVolumes(owner ...string) map[string][]types.Volume { s.mu.Lock() defer s.mu.Unlock() - if len(s.ackRows) == 0 && len(s.attemptRows) == 0 { - return nil - } - refs := map[string][]types.Volume{} - add := func(row *claimRow) { - if len(owner) > 0 && row.id != owner[0] { - return - } - for _, v := range row.refs { - if !slices.Contains(refs[row.id], v) { - refs[row.id] = append(refs[row.id], v) - } + if len(owner) > 0 { + id := owner[0] + refs := appendRecoveryRefs(slices.Clone(s.ackRefs[id]), s.attemptRefs[id]) + if len(refs) == 0 { + return nil } + return map[string][]types.Volume{id: refs} + } + return mergeRecoveryRefs(mergeRecoveryRefs(nil, s.ackRefs), s.attemptRefs) +} + +func mergeRecoveryRefs(dst, src map[string][]types.Volume) map[string][]types.Volume { + if dst == nil && len(src) > 0 { + dst = make(map[string][]types.Volume, len(src)) } - for _, row := range s.ackRows { - add(row) + for id, refs := range src { + dst[id] = appendRecoveryRefs(dst[id], refs) } - for row := range s.attemptRows { - add(row) + return dst +} + +func appendRecoveryRefs(dst, src []types.Volume) []types.Volume { + for _, v := range src { + if !slices.Contains(dst, v) { + dst = append(dst, v) + } } - return refs + return dst } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 5693007a..a5fda5ae 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -339,6 +339,7 @@ type Manager struct { volumeStat func(string) (os.FileInfo, error) // volumeAdmission counts the live holders of each volume name; guarded by m.mu. volumeAdmission map[string]volumeHolders + unknownVolumes int volumeRuns map[*types.Sandbox]*volumeRun volumeErrors map[*types.Sandbox]map[string]string diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 3bfa3cf1..50b7d614 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -70,13 +70,14 @@ func (m *Manager) Reconcile(ctx context.Context) error { owned[sb.VMName] = true referenced[sb.HibernateSnap] = true } - saveErr := m.store.save(m.claimed) + js := m.store.reset(m.claimed) pooled := make(map[string]bool, len(m.pools)) for _, p := range m.pools { m.adoptGolden(p, imageIDs[p.key.Template]) pooled[p.hash] = true } m.mu.Unlock() + saveErr := m.store.commit(js) for _, sb := range adopted { m.recordHibernate(ctx, sb) } @@ -156,6 +157,10 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec // removeStaleVM reclaims one unowned VM, reporting whether it is gone. func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRecord) bool { logger := log.WithFunc("pool.removeStaleVM") + m.mu.Lock() + m.unknownVolumes++ + m.mu.Unlock() + td := volumeTeardown{unknown: true} if rec.State == vmStateCreating { // a canceled ctx must not skip the busy check guarding the forced remove switch outcome, err := m.eng.ReconcileStaleCreate(context.WithoutCancel(ctx), name); { @@ -163,41 +168,50 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe // verb missing (cocoon < v0.5.8) or failed: keep the old sweep. logger.Warnf(ctx, "reconcile stale create %s: %v; removing", name, err) case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: + m.finishVolumeTeardown(ctx, td) return true case outcome == engine.StaleCreateBusy: logger.Infof(ctx, "stale create %s has an in-flight owner; queued for retry", name) - m.queueStaleCreate(name, rec.TapDevice()) + td = m.resolveStaleVolumes(ctx, name, td) + m.queueStaleCreate(name, rec.TapDevice(), td) return false } // not-creating: the record moved on under the lock; remove normally. } if m.removeVM(ctx, name) { + m.finishVolumeTeardown(ctx, td) return true } - m.queueRemoval(name, "", rec.TapDevice(), volumeTeardown{holds: m.holdStaleVolumes(ctx, name, rec)}, time.Time{}) + td = m.resolveStaleVolumes(ctx, name, td) + m.queueRemoval(name, "", rec.TapDevice(), td, time.Time{}) return false } -func (m *Manager) holdStaleVolumes(ctx context.Context, name string, rec types.VMRecord) []types.Volume { - if rec.State != vmStateRunning { - return nil +func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volumeTeardown) volumeTeardown { + if !td.unknown { + return td } - inspected, _, err := m.eng.Inspect(ctx, name) + inspected, present, err := m.eng.Inspect(ctx, name) if err != nil { - log.WithFunc("pool.holdStaleVolumes").Warnf(ctx, "inspect stale vm %s, its volumes stay unheld: %v", name, err) - return nil + log.WithFunc("pool.resolveStaleVolumes").Warnf(ctx, "inspect stale vm %s, keeping volume fence: %v", name, err) + return td } - holds := make([]types.Volume, len(inspected.AttachedDevices.Disks)) - for i, disk := range inspected.AttachedDevices.Disks { - holds[i].Name = disk.Name - if !disk.ReadOnly { - holds[i].Mode = types.VolumeModeRW + var holds []types.Volume + if present { + holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) + for i, disk := range inspected.AttachedDevices.Disks { + holds[i].Name = disk.Name + if !disk.ReadOnly { + holds[i].Mode = types.VolumeModeRW + } } } m.mu.Lock() m.adoptVolumes(holds) + m.unknownVolumes-- m.mu.Unlock() - return holds + td.unknown, td.holds = false, holds + return td } // resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 75efec26..5efbe253 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -45,7 +45,9 @@ func (m *Manager) confirmGone(ctx context.Context, name string) bool { // removeOrRetry reports whether the VM is confirmed gone; a survivor is queued for the reap tick. func (m *Manager) removeOrRetry(ctx context.Context, name, sandboxID string, td volumeTeardown) bool { if m.removeVM(ctx, name) { - m.finishVolumeTeardown(ctx, td) + if !m.finishVolumeTeardown(ctx, td) { + m.queueRemoval(name, sandboxID, "", td, time.Time{}) + } return true } m.queueRemoval(name, sandboxID, "", td, time.Time{}) @@ -58,9 +60,9 @@ func (m *Manager) queueRemoval(name, sandboxID, tap string, td volumeTeardown, n m.mu.Unlock() } -func (m *Manager) queueStaleCreate(name, tap string) { +func (m *Manager) queueStaleCreate(name, tap string, td volumeTeardown) { m.mu.Lock() - m.pendingRemovals[name] = pendingRemoval{tap: tap, staleCreate: true} + m.pendingRemovals[name] = pendingRemoval{tap: tap, staleCreate: true, volumes: td} m.mu.Unlock() } @@ -77,23 +79,24 @@ func (m *Manager) retryRemovals(ctx context.Context) *sync.WaitGroup { } m.mu.Unlock() names := slices.Collect(maps.Keys(batch)) - return m.runBounded(ctx, len(names), func(ctx context.Context, i int) { + return m.runBounded(context.WithoutCancel(ctx), len(names), func(ctx context.Context, i int) { m.retryRemoval(ctx, names[i], batch[names[i]]) }) } func (m *Manager) retryRemoval(ctx context.Context, name string, pending pendingRemoval) { + pending.volumes = m.resolveStaleVolumes(ctx, name, pending.volumes) if pending.staleCreate { switch outcome, err := m.eng.ReconcileStaleCreate(ctx, name); { case err != nil: log.WithFunc("pool.retryRemoval").Warnf(ctx, "reconcile stale create %s: %v; retrying", name, err) - m.queueStaleCreate(name, pending.tap) + m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateBusy: - m.queueStaleCreate(name, pending.tap) + m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: - m.finishRemoval(ctx, pending) + m.finishRemoval(ctx, name, pending) return } } @@ -101,11 +104,14 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) return } - m.finishRemoval(ctx, pending) + m.finishRemoval(ctx, name, pending) } -func (m *Manager) finishRemoval(ctx context.Context, pending pendingRemoval) { - m.finishVolumeTeardown(ctx, pending.volumes) +func (m *Manager) finishRemoval(ctx context.Context, name string, pending pendingRemoval) { + if !m.finishVolumeTeardown(ctx, pending.volumes) { + m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) + return + } if pending.sandboxID != "" { m.out.Disarm(pending.sandboxID, true) } else if pending.tap != "" { diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index b880d297..fbe73408 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -39,8 +39,10 @@ type resolvedVolume struct { // volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. type volumeTeardown struct { - holds []types.Volume - clears []string + done <-chan struct{} + unknown bool + holds []types.Volume + clears []string } type catalogVolume struct { @@ -249,7 +251,12 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res // quiesceVolumes unmounts a claim's writable mounts in reverse; a failed unmount keeps the marker. func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { + m.mu.Lock() td := volumeTeardown{holds: heldVolumes(sb)} + if run := m.volumeRuns[sb]; run != nil { + td.done = run.done + } + m.mu.Unlock() mounts := writableMounts(sb.Volumes) if mounts == 0 { return td @@ -286,9 +293,13 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT } // finishVolumeTeardown runs only once the VM is gone, when its image lock is finally released. -func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { - if len(td.holds) == 0 { - return +func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) bool { + if td.done != nil { + select { + case <-td.done: + default: + return false + } } logger := log.WithFunc("pool.finishVolumeTeardown") for _, path := range td.clears { @@ -296,7 +307,13 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { logger.Errorf(ctx, err, "clear dirty marker %s", path) } } - m.unreserveVolumes(td.holds) + m.mu.Lock() + m.releaseVolumes(td.holds) + if td.unknown { + m.unknownVolumes-- + } + m.mu.Unlock() + return true } // reserveVolumes admits one claim's volumes atomically; callers hold m.mu. @@ -308,7 +325,7 @@ func (m *Manager) reserveVolumesFor(volumes []types.Volume, owner *types.Sandbox if len(volumes) == 0 { return nil } - if len(volumes) > 0 && m.catalogPending != nil { + if m.catalogPending != nil || m.unknownVolumes > 0 { return ErrVolumeCatalogUnavailable } recovery := m.store.recoveryVolumes() @@ -443,7 +460,7 @@ func (m *Manager) volumeCatalog() map[string]*catalogVolume { // reserveResolved revalidates the binding and policy in the same decision as admission. func (m *Manager) reserveResolved(tenant string, volumes []resolvedVolume, owner *types.Sandbox) error { - if len(volumes) > 0 && m.catalogPending != nil { + if len(volumes) > 0 && (m.catalogPending != nil || m.unknownVolumes > 0) { return ErrVolumeCatalogUnavailable } var held []types.Volume diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index 1b6cc928..d10c20cb 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -60,7 +60,7 @@ func (m *Manager) loadRuntimeVolumes() error { if !st.Mode().IsRegular() || st.Mode().Perm() != 0o600 || st.Sys().(*syscall.Stat_t).Uid != 0 { return fmt.Errorf("runtime volume catalog must be a root-owned regular file with mode 0600") } - raw, err := os.ReadFile(path) + raw, err := os.ReadFile(path) //nolint:gosec // root-owned metadata below a protected directory if err != nil { return fmt.Errorf("read runtime volume catalog: %w", err) } @@ -105,6 +105,10 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo return err } m.mu.Lock() + if m.unknownVolumes > 0 { + m.mu.Unlock() + return ErrVolumeCatalogUnavailable + } if current, ok := m.runtimeVolumes[name]; ok { m.mu.Unlock() if sameRegistration(current, normalized) { @@ -131,6 +135,10 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo } m.mu.Lock() // A claim can acquire an unexplained recovery name while the backing is checked. + if m.unknownVolumes > 0 { + m.mu.Unlock() + return ErrVolumeCatalogUnavailable + } if m.volumeReferenced(name) { m.mu.Unlock() return fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) @@ -156,6 +164,10 @@ func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { return fmt.Errorf("%w: invalid volume name", ErrBadVolume) } m.mu.Lock() + if m.unknownVolumes > 0 { + m.mu.Unlock() + return ErrVolumeCatalogUnavailable + } if _, runtime := m.runtimeVolumes[name]; !runtime { _, exists := m.volumes[name] referenced := m.volumeReferenced(name) @@ -203,7 +215,11 @@ func (m *Manager) volumeReferenced(name string) bool { func (m *Manager) commitPendingCatalog(ctx context.Context) error { m.mu.Lock() proposal := m.catalogPending + unknown := m.unknownVolumes > 0 m.mu.Unlock() + if unknown { + return ErrVolumeCatalogUnavailable + } if proposal == nil { return nil } diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 4e4d489b..ba0dd9c7 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -3,7 +3,6 @@ package pool import ( "errors" "fmt" - "github.com/cocoonstack/sandbox/sandboxd/config" "os" "path/filepath" "slices" @@ -11,6 +10,7 @@ import ( "testing" "time" + "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/types" "github.com/cocoonstack/sandbox/sandboxd/utils" ) @@ -30,9 +30,9 @@ func TestRuntimeCatalogPersistsNormalizedSemantics(t *testing.T) { if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { t.Fatalf("normalized retry: %v", err) } - st, err := os.Stat(filepath.Join(m.dataDir, "runtime-volumes.json")) - if err != nil || st.Mode().Perm() != 0o600 { - t.Fatalf("metadata mode: %v %v", st, err) + st, statErr := os.Stat(filepath.Join(m.dataDir, "runtime-volumes.json")) + if statErr != nil || st.Mode().Perm() != 0o600 { + t.Fatalf("metadata mode: %v %v", st, statErr) } m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) if got := m2.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { @@ -346,6 +346,9 @@ func TestRuntimeCatalogRetirementBlocksUnregisterAndReuse(t *testing.T) { if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retirement-fenced", ""); !errors.Is(err, ErrVolumeCapture) { t.Fatalf("capture: %v", err) } + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("hibernate before retirement: %v", err) + } if err := os.Remove(tmp); err != nil { t.Fatal(err) } diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go new file mode 100644 index 00000000..e3ba30e2 --- /dev/null +++ b/sandboxd/pool/volume_cleanup_test.go @@ -0,0 +1,290 @@ +package pool + +import ( + "context" + "errors" + "slices" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestRemovedVMKeepsVolumeUntilCanceledWorkerReturns(t *testing.T) { + requireCatalogRoot(t) + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true}); err != nil { + t.Fatal(err) + } + sb := mustClaim(t, m, testKey) + eng := newGatedVolumeEngine(base) + m.eng = eng + changed := make(chan error, 1) + go func() { _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); changed <- err }() + <-eng.entered + resumed := false + defer func() { + if !resumed { + close(eng.resume) + <-changed + } + }() + released := make(chan error, 1) + go func() { released <- m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) }() + select { + case err := <-released: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("release waited for paused worker") + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { + t.Errorf("removed VM freed active worker count: %+v", got) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer overtook paused canceled worker: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Errorf("unregister overtook paused worker: %v", err) + } + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { + t.Errorf("maintenance freed active worker count: %+v", got) + } + close(eng.resume) + resumed = true + if err := <-changed; err == nil { + t.Fatal("removed claim accepted late attachment") + } + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("completed worker leaked count: %+v", got) + } + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } +} + +func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { + requireCatalogRoot(t) + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { + t.Fatal(err) + } + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if err != nil { + t.Fatal(err) + } + base.removeErrFor = sb.VMName + if err = m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("expected survivor") + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + wg := m.retryRemovals(t.Context()) + m.mu.Lock() + queued := len(m.pendingRemovals) + m.mu.Unlock() + if queued != 0 { + t.Fatal("cleanup was not dispatched") + } + err = m.UnregisterVolume(t.Context(), "data") + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + if !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("dispatched cleanup lost guard: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } +} + +type unknownVolumeEngine struct { + *fakeEngine + inspectFailed atomic.Bool + inspectEntered chan struct{} + inspectResume chan struct{} + inspectGated atomic.Bool +} + +func (e *unknownVolumeEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + if e.inspectEntered != nil && e.inspectGated.CompareAndSwap(false, true) { + close(e.inspectEntered) + <-e.inspectResume + } + if e.inspectFailed.Load() { + return types.VMRecord{}, false, errors.New("inspection unavailable") + } + return e.fakeEngine.Inspect(ctx, name) +} + +func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { + requireCatalogRoot(t) + for _, state := range []string{"running", "stopped", "creating"} { + t.Run(state, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { + t.Fatal(err) + } + name := "sbx-stale-unknown" + base.vms[name] = "/vsock/stale" + base.stopped[name] = state == "stopped" + base.creating[name] = state == "creating" + base.staleOutcome = engine.StaleCreateBusy + base.removeErrFor = name + base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + eng := &unknownVolumeEngine{fakeEngine: base} + eng.inspectFailed.Store(true) + m.eng = eng + if m.removeStaleVM(t.Context(), name, base.record(name)) { + t.Fatal("unknown stale VM unexpectedly gone") + } + for range 2 { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unknown stale attachment admitted reader: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unknown stale attachment allowed unregister: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatalf("volume-free claim blocked: %v", err) + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + wg := m.retryRemovals(t.Context()) + if err := m.RegisterVolume(t.Context(), "new", types.VolumeRegistration{Path: writeVolumeImage(t, "new", "bytes")}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("dispatched unknown fence lost: %v", err) + } + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + } + eng.inspectFailed.Store(false) + base.staleOutcome = engine.StaleCreateNotCreating + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{writers: 1}) { + t.Fatalf("inspection did not replace fence with known holds: %+v", got) + } + if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated", "bytes")}); err != nil { + t.Fatalf("resolved inspection left global fence: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("known attachment lost exclusion: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatalf("confirmed removal retained fence: %v", err) + } + }) + } +} + +func TestRuntimeCatalogLoadedBeforeClaimReconciliation(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true, Tenants: []string{"acme"}}); err != nil { + t.Fatal(err) + } + sb, claimErr := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if claimErr != nil { + t.Fatal(claimErr) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + if got := m2.Volumes("acme", nil); len(got) != 1 || got[0].Name != "data" || !got[0].Available || !got[0].Writable { + t.Fatalf("catalog absent before reconcile: %+v", got) + } + if local, err := m2.VolumePlacement(testKey, "acme", []string{"data"}); err != nil || !local { + t.Fatalf("placement before reconcile: %v %v", local, err) + } + if err := m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + view, err := m2.GetVolumes(sb.ID, "") + if err != nil || !slices.Equal(view.Volumes, sb.Volumes) { + t.Fatalf("claim recovery: %+v %v", view, err) + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("recovered holder missing: %v", err) + } +} + +func TestUnknownFencePrecedesInspectionAndClearsAfterConfirmedRemoval(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + name := "sbx-stale-before-inspect" + base.vms[name] = "/vsock/stale" + base.removeErrFor = name + eng := &unknownVolumeEngine{fakeEngine: base, inspectEntered: make(chan struct{}), inspectResume: make(chan struct{})} + eng.inspectFailed.Store(true) + m.eng = eng + done := make(chan bool, 1) + go func() { done <- m.removeStaleVM(t.Context(), name, base.record(name)) }() + <-eng.inspectEntered + if !m.mu.TryLock() { + t.Fatal("stale inspection holds manager lock") + } + m.mu.Unlock() + _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}) + close(eng.inspectResume) + if gone := <-done; gone { + t.Fatal("failed inspect/removal unexpectedly confirmed gone") + } + if !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("admitted volume while inspection was blocked: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("successful removal retained unknown fence: %v", err) + } +} + +func TestCanceledRemovalDispatchKeepsUnknownFenceOwned(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + name := "sbx-canceled-dispatch" + base.vms[name] = "/vsock/stale" + base.removeErrFor = name + eng := &unknownVolumeEngine{fakeEngine: base} + eng.inspectFailed.Store(true) + m.eng = eng + if m.removeStaleVM(t.Context(), name, base.record(name)) { + t.Fatal("expected survivor") + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + ctx, cancel := context.WithCancel(t.Context()) + wg := m.retryRemovals(ctx) + cancel() + synctest.Wait() + base.removeErrFor = "" + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("canceled dispatch stranded unknown fence: %v", err) + } + }) +} diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index e9be5c15..407c4e0b 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -100,7 +100,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v return view, err } additions := volumeDifference(target, held) - if len(additions) != 0 { + if slices.ContainsFunc(additions, func(v types.Volume) bool { return !slices.Contains(recovery, v) }) { if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { return view, err } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index ce30fd1a..c76afcc5 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -280,8 +280,11 @@ func TestLiveVolumeDoesNotBlockLifecycle(t *testing.T) { } close(resume) attachErr := <-attached - if action == "reap" { - waitFor(t, func() bool { return volumeHoldersOf(m, "data") == (volumeHolders{}) }) + if action == "release" || action == "reap" { + waitFor(t, func() bool { + m.retryRemovals(t.Context()).Wait() + return volumeHoldersOf(m, "data") == (volumeHolders{}) + }) } if action == "release" || action == "reap" { if attachErr == nil { @@ -718,6 +721,7 @@ func TestRollbackClaimTakesOverAdmittedRunHolds(t *testing.T) { run := beginAdmittedVolumeRun(t, m, sb) m.rollbackClaim(t.Context(), []*types.Sandbox{sb}) m.finishVolumeRun(sb, run) + m.retryRemovals(t.Context()).Wait() waitFor(t, m.store.synced) if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { t.Fatalf("rolled-back claim leaked its admitted hold: %+v", got) diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go index d91a687c..e5e573c2 100644 --- a/sandboxd/pool/volume_retirement_test.go +++ b/sandboxd/pool/volume_retirement_test.go @@ -2,6 +2,7 @@ package pool import ( "errors" + "fmt" "os" "path/filepath" "testing" @@ -92,7 +93,7 @@ func TestDetachedVolumeOwnRetryAndLaterAcknowledgment(t *testing.T) { if err != nil || len(view.States) != 1 || view.States[0].State != types.VolumeDetaching { t.Fatalf("unretired view: %+v, %v", view, err) } - if err := os.Remove(tmp); err != nil { + if err = os.Remove(tmp); err != nil { t.Fatal(err) } switch retry { @@ -129,6 +130,7 @@ func TestFailedClaimIntroductionAfterRenameRemainsProtected(t *testing.T) { catalog := liveCatalog(t) m := newVolumeManager(t, eng, catalog) sb := mustClaim(t, m, testKey) + m.store.sync = true m.store.recommitting.Store(true) calls := 0 m.store.replaceFile = func(raw []byte) error { @@ -202,3 +204,74 @@ func TestClaimRecoveryReferencesSurviveResetAndAreCopied(t *testing.T) { t.Fatal("empty snapshot did not retire file references") } } + +func TestClaimRecoveryAttemptsRetainOnlyDistinctReferences(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + s.replaceFile = func([]byte) error { return errors.New("no space left") } + for i := range 200 { + sb.Token = fmt.Sprint(i) + sb.Volumes[0].Mount = []string{"/data", "/other"}[i%2] + if err := s.commit(s.set(sb)); err == nil { + t.Fatal("expected failed snapshot") + } + } + if len(s.attemptRefs) != 1 || len(s.attemptRefs[sb.ID]) != 2 { + t.Fatalf("renewals retained duplicate history: %+v", s.attemptRefs) + } + refs := s.recoveryVolumes(sb.ID)[sb.ID] + refs[0].Name = "caller-copy" + if got := s.recoveryVolumes(sb.ID)[sb.ID]; len(got) != 2 || got[0].Name != "data" { + t.Fatalf("recovery references not copied: %+v", got) + } + s.replaceFile = nil + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if len(s.attemptRefs) != 0 || len(s.recoveryVolumes(sb.ID)[sb.ID]) != 1 { + t.Fatal("successful snapshot did not replace possibility history") + } +} + +func TestRecoveryOnlyRetryPreservesHeldRepairAuthorization(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{}); err == nil { + t.Fatal("expected failed detach result commit") + } + eng.liveVolumeHook = nil + if err := m.DeleteTenant(t.Context(), "acme"); err != nil { + t.Fatal(err) + } + m.mu.Lock() + entry := *m.volumes["data"] + entry.tenants = []string{"different-tenant"} + m.volumes["data"] = &entry + m.mu.Unlock() + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatalf("exact recovery retry lost held authorization: %v", err) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { + t.Fatalf("repair reservation count: %+v", got) + } + if _, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "other"}}}); !errors.Is(err, ErrTenantRemoved) { + t.Fatalf("repair exemption admitted unrelated addition: %v", err) + } +} From 9cd697f456e06f177b0f586107d04bd630891a4e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 14:34:55 +0000 Subject: [PATCH 08/72] pool: Cover incomplete stale disk inventories. --- sandboxd/pool/volume_cleanup_test.go | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index e3ba30e2..54fb7199 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -3,6 +3,9 @@ package pool import ( "context" "errors" + "fmt" + "os" + "path/filepath" "slices" "sync/atomic" "testing" @@ -13,6 +16,74 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +func TestIncompleteStaleInventoryRetainsVolumeProtection(t *testing.T) { + requireCatalogRoot(t) + for _, state := range []string{vmStateRunning, vmStateCreating, "stopped"} { + t.Run(state, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + path := writeVolumeImage(t, "incomplete-inventory", "original") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { + t.Fatal(err) + } + name := "sbx-incomplete-inventory" + base.vms[name] = "/vsock/incomplete" + base.creating[name] = state == vmStateCreating + base.staleOutcome = engine.StaleCreateBusy + base.removeErrFor = name + base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + cli := filepath.Join(t.TempDir(), "inspect") + writeInspect := func(state, inventory string) { + t.Helper() + raw := fmt.Sprintf("#!/bin/sh\nprintf '%%s\\n' '{\"config\":{\"name\":\"%s\"},\"state\":\"%s\"%s}'\n", name, state, inventory) + if err := os.WriteFile(cli, []byte(raw), 0o700); err != nil { + t.Fatal(err) + } + } + writeInspect(state, "") + m.eng = &incompleteInventoryEngine{fakeEngine: base, cli: engine.New(cli, nil, nil, false, false, "")} + rec := base.record(name) + rec.State = state + if m.removeStaleVM(t.Context(), name, rec) { + t.Fatal("stale VM unexpectedly disappeared") + } + for range 2 { + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unregister must refuse incomplete inventory: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("conflicting writer must refuse incomplete inventory: %v", err) + } + m.retryRemovals(t.Context()).Wait() + } + base.creating[name] = false + writeInspect(vmStateRunning, `,"attached_devices":{"disks":[{"name":"data","readonly":false}]}`) + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("complete inventory lost the surviving writer: %v", err) + } + other := writeVolumeImage(t, "complete-inventory-other", "other") + if err := m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: other}); err != nil { + t.Fatalf("complete inventory retained the global fence: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatalf("confirmed removal retained volume protection: %v", err) + } + }) + } +} + +type incompleteInventoryEngine struct { + *fakeEngine + cli *engine.Engine +} + +func (e *incompleteInventoryEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + return e.cli.Inspect(ctx, name) +} + func TestRemovedVMKeepsVolumeUntilCanceledWorkerReturns(t *testing.T) { requireCatalogRoot(t) base := newFakeEngine() From b343917171905eecce51c8d6d6ae0c49db1f85de Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 14:39:30 +0000 Subject: [PATCH 09/72] pool: Keep unknown volume holds for incomplete inspections. --- sandboxd/pool/reconcile.go | 4 +++- sandboxd/pool/remove.go | 2 +- sandboxd/pool/volume_cleanup_test.go | 8 ++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 50b7d614..9a753969 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -172,7 +172,6 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe return true case outcome == engine.StaleCreateBusy: logger.Infof(ctx, "stale create %s has an in-flight owner; queued for retry", name) - td = m.resolveStaleVolumes(ctx, name, td) m.queueStaleCreate(name, rec.TapDevice(), td) return false } @@ -196,6 +195,9 @@ func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volum log.WithFunc("pool.resolveStaleVolumes").Warnf(ctx, "inspect stale vm %s, keeping volume fence: %v", name, err) return td } + if present && (inspected.State != vmStateRunning || len(inspected.AttachedDevices.Disks) == 0) { + return td + } var holds []types.Volume if present { holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 5efbe253..924e3f5d 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -85,7 +85,6 @@ func (m *Manager) retryRemovals(ctx context.Context) *sync.WaitGroup { } func (m *Manager) retryRemoval(ctx context.Context, name string, pending pendingRemoval) { - pending.volumes = m.resolveStaleVolumes(ctx, name, pending.volumes) if pending.staleCreate { switch outcome, err := m.eng.ReconcileStaleCreate(ctx, name); { case err != nil: @@ -100,6 +99,7 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending return } } + pending.volumes = m.resolveStaleVolumes(ctx, name, pending.volumes) if !m.removeVM(ctx, name) { m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) return diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index 54fb7199..6a476302 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -58,6 +58,13 @@ func TestIncompleteStaleInventoryRetainsVolumeProtection(t *testing.T) { } base.creating[name] = false writeInspect(vmStateRunning, `,"attached_devices":{"disks":[{"name":"data","readonly":false}]}`) + if state == vmStateCreating { + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("busy owner must retain the unknown fence: %v", err) + } + } + base.staleOutcome = engine.StaleCreateNotCreating m.retryRemovals(t.Context()).Wait() if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { t.Errorf("complete inventory lost the surviving writer: %v", err) @@ -246,6 +253,7 @@ func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { wg.Wait() } eng.inspectFailed.Store(false) + base.stopped[name], base.creating[name] = false, false base.staleOutcome = engine.StaleCreateNotCreating m.retryRemovals(t.Context()).Wait() if got := volumeHoldersOf(m, "data"); got != (volumeHolders{writers: 1}) { From 67f6266272323750055cdf7dcb75882321de83c3 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 14:41:59 +0000 Subject: [PATCH 10/72] pool: Read catalog metadata through its directory. --- sandboxd/pool/claims.go | 1 - sandboxd/pool/volume.go | 1 - sandboxd/pool/volume_catalog.go | 10 ++++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 97ebe987..c4ae1345 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -321,7 +321,6 @@ func newClaimRow(sb *types.Sandbox) *claimRow { return &claimRow{id: sb.ID, refs: refs, dto: dtoOf(sb)} } -// recoveryVolumes includes every reference a failed replacement may have installed. func (s *claimStore) recoveryVolumes(owner ...string) map[string][]types.Volume { s.mu.Lock() defer s.mu.Unlock() diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index fbe73408..e7cba67a 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -458,7 +458,6 @@ func (m *Manager) volumeCatalog() map[string]*catalogVolume { return maps.Clone(m.volumes) } -// reserveResolved revalidates the binding and policy in the same decision as admission. func (m *Manager) reserveResolved(tenant string, volumes []resolvedVolume, owner *types.Sandbox) error { if len(volumes) > 0 && (m.catalogPending != nil || m.unknownVolumes > 0) { return ErrVolumeCatalogUnavailable diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index d10c20cb..a5b83bb8 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -60,7 +60,12 @@ func (m *Manager) loadRuntimeVolumes() error { if !st.Mode().IsRegular() || st.Mode().Perm() != 0o600 || st.Sys().(*syscall.Stat_t).Uid != 0 { return fmt.Errorf("runtime volume catalog must be a root-owned regular file with mode 0600") } - raw, err := os.ReadFile(path) //nolint:gosec // root-owned metadata below a protected directory + root, err := os.OpenRoot(m.dataDir) + if err != nil { + return fmt.Errorf("open runtime catalog directory: %w", err) + } + raw, readErr := root.ReadFile(runtimeVolumesFile) + err = errors.Join(readErr, root.Close()) if err != nil { return fmt.Errorf("read runtime volume catalog: %w", err) } @@ -134,7 +139,6 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo return err } m.mu.Lock() - // A claim can acquire an unexplained recovery name while the backing is checked. if m.unknownVolumes > 0 { m.mu.Unlock() return ErrVolumeCatalogUnavailable @@ -188,7 +192,6 @@ func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { return m.commitPendingCatalog(ctx) } -// volumeReferenced includes dispatched cleanup through its ordinary admission counts. func (m *Manager) volumeReferenced(name string) bool { if holders := m.volumeAdmission[name]; holders.readers > 0 || holders.writers > 0 { return true @@ -211,7 +214,6 @@ func (m *Manager) volumeReferenced(name string) bool { return false } -// commitPendingCatalog holds catalogWriteMu, but never m.mu across filesystem work. func (m *Manager) commitPendingCatalog(ctx context.Context) error { m.mu.Lock() proposal := m.catalogPending From 2da25a2f37ef1a2513d385cb95ccad4d14728ef7 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 14:41:59 +0000 Subject: [PATCH 11/72] sandboxd: Cover runtime catalog capability refusals. --- sandboxd/pool/pool.go | 1 + sandboxd/pool/volume_catalog_test.go | 2 +- sandboxd/server/volume_registration_test.go | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index a5fda5ae..5344d1f1 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -121,6 +121,7 @@ var ( var ( ErrVolumeRegistrationConflict = errors.New("volume registration conflicts with the catalog") ErrVolumeCatalogUnavailable = errors.New("volume catalog requires a successful persistence retry") + ErrVolumeCatalogReadOnly = errors.New("runtime volume registration requires sandboxd to run as OS root") ) // Engine is the slice of the cocoon driver the manager consumes. diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index ba0dd9c7..67fd2e25 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -233,7 +233,7 @@ func TestStaticOnlyCatalogDoesNotNeedRoot(t *testing.T) { t.Fatal("static-only startup lost catalog") } if os.Geteuid() != 0 { - if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/data"}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/data"}); !errors.Is(err, ErrVolumeCatalogReadOnly) { t.Fatalf("nonroot write: %v", err) } } diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go index ba4d4f79..8556623e 100644 --- a/sandboxd/server/volume_registration_test.go +++ b/sandboxd/server/volume_registration_test.go @@ -176,6 +176,7 @@ func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { }{ {"invalid registration", fmt.Errorf("%w: invalid directio", pool.ErrBadVolume), http.StatusBadRequest}, {"binding conflict", fmt.Errorf("%w: /private/data.img", pool.ErrVolumeRegistrationConflict), http.StatusConflict}, + {"OS root required", pool.ErrVolumeCatalogReadOnly, http.StatusConflict}, {"busy", fmt.Errorf("%w: /private/data.img", pool.ErrVolumeBusy), http.StatusConflict}, {"catalog unavailable", fmt.Errorf("%w: sync /private/runtime-volumes.json", pool.ErrVolumeCatalogUnavailable), http.StatusServiceUnavailable}, {"unexpected failure", errors.New("open /private/data.img: permission denied"), http.StatusInternalServerError}, @@ -201,6 +202,8 @@ func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { if err != nil || seconds < 1 || !strings.Contains(got["error"], "retry") { t.Fatalf("503 missing retry guidance: headers=%v body=%s", resp.Header, body) } + } else if resp.Header.Get("Retry-After") != "" { + t.Fatalf("permanent refusal advertises retry: %v", resp.Header) } }) } From 8afcb15d24905c805d5b7303c5746cbc99da31da Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sat, 24 Oct 2026 14:45:02 +0000 Subject: [PATCH 12/72] sandboxd: Distinguish read-only catalog capability from retries. --- docs/sandboxd-api.md | 11 +++++++++++ sandboxd/pool/volume_catalog.go | 4 ++-- sandboxd/server/server.go | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 12c0785e..5ccd6a2b 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -289,6 +289,9 @@ Only registration metadata is persisted locally in is root-owned with mode 0600, and its directory must protect the record. Runtime mutations require sandboxd to run as OS root. The backing file does not require root ownership. Registration creates, grows, and deletes no disks. +An OS-nonroot daemon refuses runtime mutations with 409 and no `Retry-After`. +Malformed or conflicting metadata prevents startup. A temporarily missing +backing file keeps its registration and refuses new attachment. Catalog replacement and synchronization are independent of `sync_claims`, which controls claim snapshot synchronization. Process-restart checks do not prove power-loss recovery. The catalog is node-local with one sandboxd writer. @@ -314,6 +317,14 @@ verification is included here. Downgrading to a daemon without runtime registration requires draining runtime references or supplying equivalent static volume config. Pending live volume changes must finish before downgrade. +An unresolved stale VM keeps volume admission and catalog changes fenced while +cleanup retries continue. Inspect success alone does not prove a complete disk +inventory. A present VM with omitted disks, or an active creating owner, retains +that fence until a complete running inventory or confirmed removal resolves it. +An empty inventory is indistinguishable from an omitted one and stays fenced. +Volume-free claims continue to work. Path uniqueness uses cleaned path strings; +the operator coordinates symlink and hardlink aliases and cross-node ownership. + ## GET /v1/sandboxes/{id}/volumes Returns the persisted desired set (`volumes_want`), confirmed applied set diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index a5b83bb8..254ffc3a 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -98,7 +98,7 @@ func (m *Manager) loadRuntimeVolumes() error { // RegisterVolume durably binds a root-managed name to an existing backing file. func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { if os.Geteuid() != 0 { - return fmt.Errorf("%w: runtime catalog mutation requires effective uid zero", ErrVolumeCatalogUnavailable) + return ErrVolumeCatalogReadOnly } m.catalogWriteMu.Lock() defer m.catalogWriteMu.Unlock() @@ -157,7 +157,7 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo // UnregisterVolume retires an unreferenced runtime binding without touching its backing. func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { if os.Geteuid() != 0 { - return fmt.Errorf("%w: runtime catalog mutation requires effective uid zero", ErrVolumeCatalogUnavailable) + return ErrVolumeCatalogReadOnly } m.catalogWriteMu.Lock() defer m.catalogWriteMu.Unlock() diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 70279f63..4ad768a6 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -48,6 +48,7 @@ var poolErrHTTP = []struct { {pool.ErrBadKey, http.StatusBadRequest, ""}, {pool.ErrBadVolume, http.StatusBadRequest, ""}, {pool.ErrVolumeRegistrationConflict, http.StatusConflict, pool.ErrVolumeRegistrationConflict.Error()}, + {pool.ErrVolumeCatalogReadOnly, http.StatusConflict, pool.ErrVolumeCatalogReadOnly.Error()}, {pool.ErrVolumeCatalogUnavailable, http.StatusServiceUnavailable, "volume catalog unavailable; retry"}, {pool.ErrBadName, http.StatusBadRequest, ""}, {pool.ErrBadCount, http.StatusBadRequest, ""}, From 89b372b3b65be30974b8184623342ad16d5428cb Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 00:28:07 +0000 Subject: [PATCH 13/72] server: Require configured authentication for volume registration. --- sandboxd/server/volume_registration.go | 17 ++++++-- sandboxd/server/volume_registration_test.go | 45 +++++++++++++++++++-- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/sandboxd/server/volume_registration.go b/sandboxd/server/volume_registration.go index 80de93e4..39bad6a6 100644 --- a/sandboxd/server/volume_registration.go +++ b/sandboxd/server/volume_registration.go @@ -1,6 +1,7 @@ package server import ( + "context" "errors" "net/http" @@ -9,6 +10,10 @@ import ( ) func (s *Server) handleVolumeRegistration(w http.ResponseWriter, r *http.Request) { + if s.apiToken == "" { + writeErr(w, http.StatusForbidden, "volume registration requires a configured api token") + return + } name := r.PathValue("name") if !types.ValidVolumeName(name) { writeErr(w, http.StatusBadRequest, "invalid volume name") @@ -26,8 +31,14 @@ func (s *Server) handleVolumeRegistration(w http.ResponseWriter, r *http.Request } else { err = s.mgr.UnregisterVolume(r.Context(), name) } - if errors.Is(err, pool.ErrVolumeBusy) { - err = pool.ErrVolumeBusy + switch { + case errors.Is(err, pool.ErrVolumeCatalogUnavailable): + writePoolErr(w, err) + case errors.Is(err, context.DeadlineExceeded): + writeErr(w, http.StatusGatewayTimeout, op+" timed out") + case errors.Is(err, context.Canceled): + writeErr(w, http.StatusRequestTimeout, op+" canceled") + default: + writeResult(w, r, op, name, op+" failed", err, noContent(w)) } - writeResult(w, r, op, name, op+" failed", err, noContent(w)) } diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go index 8556623e..9916ce8c 100644 --- a/sandboxd/server/volume_registration_test.go +++ b/sandboxd/server/volume_registration_test.go @@ -1,6 +1,7 @@ package server import ( + "bytes" "context" "encoding/json/v2" "errors" @@ -8,6 +9,8 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" "reflect" "strconv" "strings" @@ -29,8 +32,8 @@ func TestVolumeRegistrationRoutesRequireRoot(t *testing.T) { {"sandbox", "root", "sandbox-token", http.StatusUnauthorized}, {"unknown", "root", "unknown", http.StatusUnauthorized}, {"missing", "root", "", http.StatusUnauthorized}, - {"open without token", "", "", http.StatusNoContent}, - {"open with tenant token", "", "acme-token", http.StatusNoContent}, + {"open without token", "", "", http.StatusForbidden}, + {"open with tenant token", "", "acme-token", http.StatusForbidden}, } { t.Run(method+"/"+tt.name, func(t *testing.T) { called := false @@ -57,6 +60,39 @@ func TestVolumeRegistrationRoutesRequireRoot(t *testing.T) { } } +func TestVolumeRegistrationRequiresConfiguredTokenBeforeReadingBody(t *testing.T) { + path := filepath.Join(t.TempDir(), "data.img") + if err := os.WriteFile(path, []byte("existing backing"), 0600); err != nil { + t.Fatal(err) + } + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, token := range []string{"", "acme-token", "unknown"} { + t.Run(method+"/"+token, func(t *testing.T) { + called := false + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, + unregisterVolume: func(context.Context, string) error { called = true; return nil }, + } + handler := New("", "", mgr, nil, nil, nil, nil, nil).Handler() + body := bytes.NewBufferString(fmt.Sprintf(`{"path":%q,"writable":true}`, path)) + bodyLen := body.Len() + req := httptest.NewRequest(method, "/v1/volumes/data", body) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusForbidden || called || body.Len() != bodyLen { + t.Fatalf("status=%d called=%v unread=%d/%d, want 403 without reading body or calling manager", w.Code, called, body.Len(), bodyLen) + } + if w.Header().Get("Retry-After") != "" { + t.Fatalf("permanent refusal advertises retry: %v", w.Header()) + } + }) + } + } +} + func TestVolumeRegistrationRoutesForwardBindingAndContext(t *testing.T) { type requestKey struct{} const body = `{"path":"/private/images/../data.img","directio":"auto","writable":true,"tenants":["beta","acme"]}` @@ -177,8 +213,11 @@ func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { {"invalid registration", fmt.Errorf("%w: invalid directio", pool.ErrBadVolume), http.StatusBadRequest}, {"binding conflict", fmt.Errorf("%w: /private/data.img", pool.ErrVolumeRegistrationConflict), http.StatusConflict}, {"OS root required", pool.ErrVolumeCatalogReadOnly, http.StatusConflict}, - {"busy", fmt.Errorf("%w: /private/data.img", pool.ErrVolumeBusy), http.StatusConflict}, {"catalog unavailable", fmt.Errorf("%w: sync /private/runtime-volumes.json", pool.ErrVolumeCatalogUnavailable), http.StatusServiceUnavailable}, + {"uncertain cancellation", fmt.Errorf("%w: %w", pool.ErrVolumeCatalogUnavailable, context.Canceled), http.StatusServiceUnavailable}, + {"uncertain timeout", fmt.Errorf("%w: %w", pool.ErrVolumeCatalogUnavailable, context.DeadlineExceeded), http.StatusServiceUnavailable}, + {"timeout", fmt.Errorf("register: %w", context.DeadlineExceeded), http.StatusGatewayTimeout}, + {"canceled", fmt.Errorf("register: %w", context.Canceled), http.StatusRequestTimeout}, {"unexpected failure", errors.New("open /private/data.img: permission denied"), http.StatusInternalServerError}, } { t.Run(method+"/"+tt.name, func(t *testing.T) { From 6ef7d5caca2696978d7d5be75547f0e2ab2b741d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 00:52:29 +0000 Subject: [PATCH 14/72] pool: reject invalid catalog writes before admission fencing --- .github/workflows/sandboxd.yml | 9 + docs/live-volumes.md | 2 +- docs/sandboxd-api.md | 43 ++- docs/security.md | 12 +- sandboxd/config/config.go | 5 + sandboxd/config/config_test.go | 13 + sandboxd/pool/pool.go | 38 +-- sandboxd/pool/reconcile.go | 4 +- sandboxd/pool/volume.go | 69 ++--- sandboxd/pool/volume_catalog.go | 319 ++++++++++++-------- sandboxd/pool/volume_catalog_test.go | 278 ++++++++++++++++- sandboxd/pool/volume_cleanup_test.go | 116 +++++++ sandboxd/pool/volume_convergence_test.go | 6 +- sandboxd/pool/volume_retirement_test.go | 45 ++- sandboxd/pool/volume_rw_test.go | 4 +- sandboxd/server/server.go | 1 + sandboxd/server/volume_registration_test.go | 3 +- 17 files changed, 753 insertions(+), 214 deletions(-) diff --git a/.github/workflows/sandboxd.yml b/.github/workflows/sandboxd.yml index fdbba839..c96b1d73 100644 --- a/.github/workflows/sandboxd.yml +++ b/.github/workflows/sandboxd.yml @@ -67,6 +67,15 @@ jobs: - name: Test run: make go-test + - name: Test runtime catalog as root + working-directory: sandboxd + shell: bash + run: | + GOWORK=off go test -race -c -o "$RUNNER_TEMP/catalog.test" ./pool + sudo "$RUNNER_TEMP/catalog.test" -test.count=1 -test.v -test.run 'Test(RuntimeCatalog|RuntimeRegistration|ResolvedCatalog|IncompleteStaleInventory|RemovedVMKeepsVolume|DispatchedCleanup|UnknownStaleVolumes)' | tee "$RUNNER_TEMP/catalog-test.log" + grep -q '^--- PASS: TestRuntimeCatalogPersistsNormalizedSemantics ' "$RUNNER_TEMP/catalog-test.log" + ! grep -- '--- SKIP' "$RUNNER_TEMP/catalog-test.log" + - name: Test the PostgreSQL meta store working-directory: sandboxd shell: bash diff --git a/docs/live-volumes.md b/docs/live-volumes.md index c4982662..19dac036 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -9,7 +9,7 @@ file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. A sandbox token alone cannot manage volumes. -The node root token can [register existing backing files](sandboxd-api.md#put-v1volumesname) +A configured node root token can [register existing regular backing files](sandboxd-api.md#put-v1volumesname) without editing static config. Registration persists only node-local catalog metadata and does not manage disk files. For a sleep and wake workflow, an external controller owns quiescence and unmount before detach, a reservation diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 5ccd6a2b..44243500 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -257,9 +257,9 @@ entry's response is byte-identical to v1. ## PUT /v1/volumes/{name} -Auth: root-only node API token. An empty configured `api_token` preserves the -existing open-root mode. A tenant token returns 403 when authentication is -enabled. Sandbox, unknown, and missing tokens return 401. +Auth: the configured root node API token. With no configured `api_token`, +registration and unregistration return 403. A tenant token returns 403. +Sandbox, unknown, and missing tokens return 401 when `api_token` is configured. Registers an existing backing file on this node. The URL supplies the name; the body has no `name` field. Unknown fields, duplicate fields, and trailing @@ -272,21 +272,37 @@ JSON return 400. - `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. - `path` uses the same absolute-path validation as static volume config. + Registration also requires an existing regular file. A symlink to a regular + file is allowed; directories and device files return 400. - `directio` is `on`, `off`, or `auto`, and defaults to `off`. - `writable` defaults to `false` and controls writable claim admission. - `tenants` limits use to the named tenants. Omitted or empty permits all tenants. + Names use the tenant API name syntax. A valid name may be registered before + that tenant exists; it grants no access until the tenant is created. Success is 204 with an empty body. Repeating the same name, cleaned path, and normalized registration fields succeeds. A different binding or different registration fields return 409. Static and runtime registrations cannot share a name or a cleaned backing path. Invalid registration fields return 400. -Unresolved catalog persistence returns 503 with `Retry-After`; retry the request. +Invalid catalog directory or file ownership, permissions, or type returns 409 +without `Retry-After`, before accepting a new write. This refusal does not block +claims using the existing catalog. The storage configuration must be repaired +before that mutation can succeed. + +An uncertain write returns 503 with `Retry-After`. The requested registration +may still take effect through background retry or a later PUT or DELETE, +including a request for another name. A 503 can also reject a request before +acceptance, so it does not promise that the request will take effect. After +repairing storage failures, retry the exact request to establish completion. +Cancellation is not a rollback once a write has been accepted. GET retains its existing projection without host paths or tenant access lists. Registration does not expand live mutation beyond mounted read-only volumes. Only registration metadata is persisted locally in `data_dir/runtime-volumes.json`, which loads before claim recovery. The record -is root-owned with mode 0600, and its directory must protect the record. +is a root-owned regular file with mode 0600. The data directory and every +ancestor must be root-owned directories, not symlinks. Group-writable or +world-writable directories must have the sticky bit set. Runtime mutations require sandboxd to run as OS root. The backing file does not require root ownership. Registration creates, grows, and deletes no disks. An OS-nonroot daemon refuses runtime mutations with 409 and no `Retry-After`. @@ -295,6 +311,10 @@ backing file keeps its registration and refuses new attachment. Catalog replacement and synchronization are independent of `sync_claims`, which controls claim snapshot synchronization. Process-restart checks do not prove power-loss recovery. The catalog is node-local with one sandboxd writer. +Startup reads only `runtime-volumes.json`. Crash leftovers named +`runtime-volumes.json.*.tmp` are neither loaded nor promoted. An operator can +remove those leftovers after stopping sandboxd and preserving any needed +evidence. ## DELETE /v1/volumes/{name} @@ -303,8 +323,10 @@ never deletes or changes the backing file. Success, including an absent, unreferenced runtime name, is 204 with an empty body. Static registrations and names referenced by a claim, an in-flight operation, cleanup, or an unacknowledged recovery snapshot return 409. Catalog persistence uncertainty -returns 503 with `Retry-After`; retry the request. An empty GET volume view alone -does not establish that a recovery reference has retired. +returns 503 with `Retry-After` and has the same asynchronous completion +semantics as PUT. A failed DELETE may therefore remove the registration later. +An empty GET volume view alone does not establish that a recovery reference +has retired. An external controller owns quiescence and guest unmount before detach. It confirms successful detach and recovery bookkeeping before hibernation, and @@ -322,8 +344,11 @@ cleanup retries continue. Inspect success alone does not prove a complete disk inventory. A present VM with omitted disks, or an active creating owner, retains that fence until a complete running inventory or confirmed removal resolves it. An empty inventory is indistinguishable from an omitted one and stays fenced. -Volume-free claims continue to work. Path uniqueness uses cleaned path strings; -the operator coordinates symlink and hardlink aliases and cross-node ownership. +A warm VM name is not evidence that no volume is attached; attachment can +succeed before claim finalization fails. Known inventories keep holds on their +reported names. Volume-free claims continue to work. Path uniqueness uses +cleaned path strings. The operator coordinates symlink and hardlink aliases +and cross-node ownership. ## GET /v1/sandboxes/{id}/volumes diff --git a/docs/security.md b/docs/security.md index 6c519f49..5808c3ad 100644 --- a/docs/security.md +++ b/docs/security.md @@ -96,13 +96,19 @@ The volume catalog is an operator-owned data boundary. A volume name and its access list must mean the same thing fleet-wide, although membership is node-local. An empty entry `tenants` list permits every authenticated scope; a nonempty list permits only those named tenants, while the root token always -has access. Config load rejects an access-list name that is not a configured -tenant. Claim lookup returns byte-identical errors for an unknown and a -forbidden name, and catalog discovery filters before replying, so a tenant +has access. Access-list names must use valid tenant name syntax. A name that +does not yet identify a tenant grants no access until that tenant exists. +Claim lookup returns byte-identical errors for an unknown and a forbidden name, and catalog discovery filters before replying, so a tenant cannot enumerate restricted entries by probing. Gossip carries names only. Neither gossip, discovery, persisted claims, usage events, nor the sandbox index exposes host image paths or access lists. +Runtime registration and unregistration require a configured root `api_token`; +these endpoints return 403 when the token is unset. The daemon must also run +as OS root to change the root-owned registration file. Runtime registration +accepts existing regular files, including symlinks to regular files. Operators +remain responsible for selecting backing paths and coordinating aliases. + Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path to every destination its tenant egress policy permits; review the volume access diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index d2b2f7dd..a70c4fa3 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -423,6 +423,11 @@ func ValidateVolumeCatalog(volumes []VolumeSpec) error { if !types.ValidDirectIO(volume.DirectIO) { return fmt.Errorf("volume %q directio must be on, off, or auto, got %q", volume.Name, volume.DirectIO) } + for _, tenant := range volume.Tenants { + if !types.NameRe.MatchString(tenant) { + return fmt.Errorf("volume %q tenant %q must match %s", volume.Name, tenant, types.NameRe) + } + } } return nil } diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 06c4fe3c..dd524fe0 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -471,3 +471,16 @@ func writeConfig(t *testing.T, body string) string { } return path } + +func TestValidateVolumeCatalogTenantNames(t *testing.T) { + for _, tenant := range []string{"", "has space", "_leading", strings.Repeat("a", 64)} { + spec := VolumeSpec{Name: "data", Path: "/images/data", DirectIO: types.DirectIOOff, Tenants: []string{tenant}} + if err := ValidateVolumeCatalog([]VolumeSpec{spec}); err == nil { + t.Errorf("invalid tenant %q accepted", tenant) + } + } + spec := VolumeSpec{Name: "data", Path: "/images/data", DirectIO: types.DirectIOOff, Tenants: []string{"future-tenant", "team/project:v1"}} + if err := ValidateVolumeCatalog([]VolumeSpec{spec}); err != nil { + t.Fatalf("syntactically valid future tenant rejected: %v", err) + } +} diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 5344d1f1..52348967 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -122,6 +122,7 @@ var ( ErrVolumeRegistrationConflict = errors.New("volume registration conflicts with the catalog") ErrVolumeCatalogUnavailable = errors.New("volume catalog requires a successful persistence retry") ErrVolumeCatalogReadOnly = errors.New("runtime volume registration requires sandboxd to run as OS root") + ErrVolumeCatalogInvalid = errors.New("runtime volume catalog storage configuration is invalid") ) // Engine is the slice of the cocoon driver the manager consumes. @@ -327,22 +328,14 @@ type PeerDeleteFunc func(ctx context.Context, id string) // Manager owns the node's pools, claims, and their persistence. type Manager struct { - eng Engine - dataDir string - egress bool - maxFork int - store *claimStore - volumes map[string]*catalogVolume - runtimeVolumes map[string]types.VolumeRegistration - catalogPending map[string]types.VolumeRegistration - catalogWriteMu sync.Mutex - catalogReplace func(string, []byte) error - volumeStat func(string) (os.FileInfo, error) - // volumeAdmission counts the live holders of each volume name; guarded by m.mu. - volumeAdmission map[string]volumeHolders - unknownVolumes int - volumeRuns map[*types.Sandbox]*volumeRun - volumeErrors map[*types.Sandbox]map[string]string + eng Engine + dataDir string + egress bool + maxFork int + store *claimStore + catalog volumeCatalog + volumeRuns map[*types.Sandbox]*volumeRun + volumeErrors map[*types.Sandbox]map[string]string poolStore *poolStore // poolShared is the cell's pool set with a meta store; poolVersion, under m.mu, is its newest version applied. @@ -460,8 +453,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg cocoondSocket: cfg.CocoondSocket, maxFork: cmp.Or(cfg.MaxForkCount, defaultMaxFork), store: newClaimStore(cfg.DataDir, cfg.SyncClaims), - volumes: make(map[string]*catalogVolume, len(cfg.Volumes)), - volumeAdmission: map[string]volumeHolders{}, + catalog: volumeCatalog{entries: make(map[string]*catalogVolume, len(cfg.Volumes)), holders: map[string]volumeHolders{}}, volumeRuns: map[*types.Sandbox]*volumeRun{}, volumeErrors: map[*types.Sandbox]map[string]string{}, poolStore: newPoolStore(cfg.DataDir), @@ -490,10 +482,12 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg healSem: make(chan struct{}, maxConcurrentHeals), } for _, volume := range cfg.Volumes { - m.volumes[volume.Name] = &catalogVolume{ - disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: volume.DirectIO}, - tenants: slices.Clone(volume.Tenants), - writable: volume.Writable, + m.catalog.generation++ + m.catalog.entries[volume.Name] = &catalogVolume{ + generation: m.catalog.generation, + disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: cmp.Or(volume.DirectIO, types.DirectIOOff)}, + tenants: slices.Clone(volume.Tenants), + writable: volume.Writable, } } if err := m.loadRuntimeVolumes(); err != nil { diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 9a753969..7b1510b9 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -158,7 +158,7 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRecord) bool { logger := log.WithFunc("pool.removeStaleVM") m.mu.Lock() - m.unknownVolumes++ + m.catalog.unknown++ m.mu.Unlock() td := volumeTeardown{unknown: true} if rec.State == vmStateCreating { @@ -210,7 +210,7 @@ func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volum } m.mu.Lock() m.adoptVolumes(holds) - m.unknownVolumes-- + m.catalog.unknown-- m.mu.Unlock() td.unknown, td.holds = false, holds return td diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index e7cba67a..2362ab22 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -32,9 +32,9 @@ type volumeHolders struct { } type resolvedVolume struct { - entry *catalogVolume - disk engine.VolumeSpec - applied types.Volume + generation uint64 + disk engine.VolumeSpec + applied types.Volume } // volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. @@ -46,9 +46,11 @@ type volumeTeardown struct { } type catalogVolume struct { - disk engine.VolumeSpec - tenants []string - writable bool + generation uint64 + runtime bool + disk engine.VolumeSpec + tenants []string + writable bool } func (v catalogVolume) allowed(tenant string) bool { @@ -57,7 +59,7 @@ func (v catalogVolume) allowed(tenant string) bool { // Volumes reports the caller-visible fleet catalog; an empty tenant means root. func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeInfo { - catalog := m.volumeCatalog() + catalog := m.catalogSnapshot() infos := make([]types.VolumeInfo, 0, len(catalog)) for name, volume := range catalog { if !volume.allowed(tenant) { @@ -94,7 +96,7 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI // VolumeNames returns the sorted names this node can currently serve. func (m *Manager) VolumeNames() []string { - catalog := m.volumeCatalog() + catalog := m.catalogSnapshot() names := make([]string, 0, len(catalog)) for name, volume := range catalog { if _, err := m.statVolume(volume.disk.Path); err == nil { @@ -114,7 +116,7 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri return true, nil } local := true - catalog := m.volumeCatalog() + catalog := m.catalogSnapshot() for _, name := range names { volume, ok := catalog[name] if !ok { @@ -144,11 +146,11 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } m.mu.Lock() - if m.catalogPending != nil { + if err := m.catalog.admissionError(); err != nil { m.mu.Unlock() - return nil, ErrVolumeCatalogUnavailable + return nil, err } - catalog := maps.Clone(m.volumes) + catalog := maps.Clone(m.catalog.entries) m.mu.Unlock() resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { @@ -167,12 +169,8 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) (resolvedVolume, error) { m.mu.Lock() - entry := m.volumes[volume.Name] - pending := m.catalogPending != nil + entry := m.catalog.entries[volume.Name] m.mu.Unlock() - if pending && entry == nil { - return resolvedVolume{}, ErrVolumeCatalogUnavailable - } return m.resolveCatalogEntry(ctx, volume, entry) } @@ -191,7 +189,7 @@ func (m *Manager) resolveCatalogEntry(ctx context.Context, volume types.Volume, disk := entry.disk disk.RW = volume.RW() volume.AttachOnly = false // past validation the empty mount carries it - return resolvedVolume{entry: entry, disk: disk, applied: volume}, nil + return resolvedVolume{generation: entry.generation, disk: disk, applied: volume}, nil } // confirmVolumesClean refuses a read-only claim of a dirty image; run after holds are taken. @@ -276,7 +274,7 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT continue } m.mu.Lock() - if entry, ok := m.volumes[volume.Name]; ok { + if entry, ok := m.catalog.entries[volume.Name]; ok { td.clears = append(td.clears, entry.disk.Path) } m.mu.Unlock() @@ -310,7 +308,7 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) b m.mu.Lock() m.releaseVolumes(td.holds) if td.unknown { - m.unknownVolumes-- + m.catalog.unknown-- } m.mu.Unlock() return true @@ -325,8 +323,8 @@ func (m *Manager) reserveVolumesFor(volumes []types.Volume, owner *types.Sandbox if len(volumes) == 0 { return nil } - if m.catalogPending != nil || m.unknownVolumes > 0 { - return ErrVolumeCatalogUnavailable + if err := m.catalog.admissionError(); err != nil { + return err } recovery := m.store.recoveryVolumes() for _, volume := range volumes { @@ -340,7 +338,7 @@ func (m *Manager) reserveVolumesFor(volumes []types.Volume, owner *types.Sandbox } } } - holders := m.volumeAdmission[volume.Name] + holders := m.catalog.holders[volume.Name] if holders.writers > 0 || (volume.RW() && holders.readers > 0) { return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) } @@ -352,30 +350,30 @@ func (m *Manager) reserveVolumesFor(volumes []types.Volume, owner *types.Sandbox // adoptVolumes counts volumes an adopted claim already holds; callers hold m.mu. func (m *Manager) adoptVolumes(volumes []types.Volume) { for _, volume := range volumes { - holders := m.volumeAdmission[volume.Name] + holders := m.catalog.holders[volume.Name] if volume.RW() { holders.writers++ } else { holders.readers++ } - m.volumeAdmission[volume.Name] = holders + m.catalog.holders[volume.Name] = holders } } // releaseVolumes drops one claim's admission holds; callers hold m.mu. func (m *Manager) releaseVolumes(volumes []types.Volume) { for _, volume := range volumes { - holders := m.volumeAdmission[volume.Name] + holders := m.catalog.holders[volume.Name] if volume.RW() { holders.writers-- } else { holders.readers-- } if holders.writers < 1 && holders.readers < 1 { - delete(m.volumeAdmission, volume.Name) + delete(m.catalog.holders, volume.Name) continue } - m.volumeAdmission[volume.Name] = holders + m.catalog.holders[volume.Name] = holders } } @@ -452,23 +450,20 @@ func syncDir(dir string) error { return errors.Join(d.Sync(), d.Close()) } -func (m *Manager) volumeCatalog() map[string]*catalogVolume { +func (m *Manager) catalogSnapshot() map[string]*catalogVolume { m.mu.Lock() defer m.mu.Unlock() - return maps.Clone(m.volumes) + return maps.Clone(m.catalog.entries) } func (m *Manager) reserveResolved(tenant string, volumes []resolvedVolume, owner *types.Sandbox) error { - if len(volumes) > 0 && (m.catalogPending != nil || m.unknownVolumes > 0) { - return ErrVolumeCatalogUnavailable - } var held []types.Volume if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { held = slices.Concat(heldVolumes(owner), m.store.recoveryVolumes(owner.ID)[owner.ID]) } for _, v := range volumes { - entry := m.volumes[v.applied.Name] - if entry == nil || entry != v.entry { + entry := m.catalog.entries[v.applied.Name] + if entry == nil || entry.generation != v.generation { return ErrVolumeUnavailable } if !slices.Contains(held, v.applied) && !entry.allowed(tenant) { @@ -482,8 +477,8 @@ func (m *Manager) reserveResolved(tenant string, volumes []resolvedVolume, owner } func (m *Manager) statVolume(path string) (os.FileInfo, error) { - if m.volumeStat != nil { - return m.volumeStat(path) + if m.catalog.stat != nil { + return m.catalog.stat(path) } return os.Stat(path) } diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index 254ffc3a..c73d8b35 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -6,9 +6,11 @@ import ( "errors" "fmt" "maps" + "math" "os" "path/filepath" "slices" + "sync" "syscall" "github.com/projecteru2/core/log" @@ -21,16 +23,74 @@ import ( const runtimeVolumesFile = "runtime-volumes.json" -type runtimeCatalog struct { - Version int `json:"version"` - Volumes map[string]types.VolumeRegistration `json:"volumes"` +type ( + runtimeCatalog struct { + Version int `json:"version"` + Volumes map[string]types.VolumeRegistration `json:"volumes"` + } + + volumeCatalog struct { + entries map[string]*catalogVolume + generation uint64 + pending *catalogWrite + holders map[string]volumeHolders + unknown int + writeMu sync.Mutex + replace func(string, []byte) error + stat func(string) (os.FileInfo, error) + } + + catalogWrite struct { + entries map[string]*catalogVolume + generation uint64 + raw []byte + } +) + +func (c *volumeCatalog) admissionError() error { + if c.pending != nil || c.unknown > 0 { + return ErrVolumeCatalogUnavailable + } + return nil +} + +func (v catalogVolume) registration() types.VolumeRegistration { + return types.VolumeRegistration{Path: v.disk.Path, DirectIO: v.disk.DirectIO, Writable: v.writable, Tenants: v.tenants} +} + +func catalogDefinitions(entries map[string]*catalogVolume) []config.VolumeSpec { + volumes := make([]config.VolumeSpec, 0, len(entries)) + for name, entry := range entries { + volumes = append(volumes, config.VolumeSpec{ + Name: name, Path: entry.disk.Path, DirectIO: entry.disk.DirectIO, + Writable: entry.writable, Tenants: entry.tenants, + }) + } + return volumes +} + +func (c *volumeCatalog) adding(name string, spec types.VolumeRegistration) (*catalogWrite, error) { + if c.generation == math.MaxUint64 { + return nil, fmt.Errorf("%w: registration generations exhausted", ErrVolumeRegistrationConflict) + } + entries := maps.Clone(c.entries) + entries[name] = &catalogVolume{ + disk: engine.VolumeSpec{Name: name, Path: spec.Path, DirectIO: spec.DirectIO}, + tenants: spec.Tenants, writable: spec.Writable, runtime: true, generation: c.generation + 1, + } + if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { + return nil, fmt.Errorf("%w: %w", ErrVolumeRegistrationConflict, err) + } + return &catalogWrite{entries: entries, generation: c.generation + 1}, nil } func normalizeRegistration(name string, spec types.VolumeRegistration) (types.VolumeRegistration, error) { if spec.DirectIO == "" { spec.DirectIO = types.DirectIOOff } - if err := config.ValidateVolumeCatalog([]config.VolumeSpec{{Name: name, Path: spec.Path, DirectIO: spec.DirectIO}}); err != nil { + if err := config.ValidateVolumeCatalog([]config.VolumeSpec{{ + Name: name, Path: spec.Path, DirectIO: spec.DirectIO, Writable: spec.Writable, Tenants: spec.Tenants, + }}); err != nil { return spec, fmt.Errorf("%w: %w", ErrBadVolume, err) } spec.Path = filepath.Clean(spec.Path) @@ -45,28 +105,20 @@ func sameRegistration(a, b types.VolumeRegistration) bool { } func (m *Manager) loadRuntimeVolumes() error { - m.runtimeVolumes = map[string]types.VolumeRegistration{} - path := filepath.Join(m.dataDir, runtimeVolumesFile) - st, err := os.Lstat(path) - if errors.Is(err, os.ErrNotExist) { + if _, err := os.Lstat(filepath.Join(m.dataDir, runtimeVolumesFile)); errors.Is(err, os.ErrNotExist) { return nil - } - if err != nil { + } else if err != nil { return fmt.Errorf("inspect runtime volume catalog: %w", err) } - if err = protectedCatalogPath(m.dataDir); err != nil { + if err := validateRuntimeCatalogPath(m.dataDir); err != nil { return err } - if !st.Mode().IsRegular() || st.Mode().Perm() != 0o600 || st.Sys().(*syscall.Stat_t).Uid != 0 { - return fmt.Errorf("runtime volume catalog must be a root-owned regular file with mode 0600") - } root, err := os.OpenRoot(m.dataDir) if err != nil { return fmt.Errorf("open runtime catalog directory: %w", err) } raw, readErr := root.ReadFile(runtimeVolumesFile) - err = errors.Join(readErr, root.Close()) - if err != nil { + if err = errors.Join(readErr, root.Close()); err != nil { return fmt.Errorf("read runtime volume catalog: %w", err) } var record runtimeCatalog @@ -74,24 +126,31 @@ func (m *Manager) loadRuntimeVolumes() error { return fmt.Errorf("parse runtime volume catalog: %w", err) } if record.Version != 1 || record.Volumes == nil { - return fmt.Errorf("invalid runtime volume catalog version or volumes") - } - merged := make([]config.VolumeSpec, 0, len(m.volumes)+len(record.Volumes)) - for name, entry := range m.volumes { - merged = append(merged, config.VolumeSpec{Name: name, Path: entry.disk.Path, DirectIO: entry.disk.DirectIO}) + return errors.New("invalid runtime volume catalog version or volumes") } + entries := maps.Clone(m.catalog.entries) + generation := m.catalog.generation for name, spec := range record.Volumes { - normalized, err := normalizeRegistration(name, spec) - if err != nil { - return fmt.Errorf("runtime volume catalog: %w", err) + normalized, normalizeErr := normalizeRegistration(name, spec) + if normalizeErr != nil { + return fmt.Errorf("runtime volume catalog: %w", normalizeErr) + } + if m.catalog.entries[name] != nil { + return fmt.Errorf("runtime volume catalog duplicates static name %q", name) + } + if generation == math.MaxUint64 { + return errors.New("runtime volume catalog generations exhausted") + } + generation++ + entries[name] = &catalogVolume{ + disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, + tenants: normalized.Tenants, writable: normalized.Writable, runtime: true, generation: generation, } - m.runtimeVolumes[name] = normalized - merged = append(merged, config.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}) } - if err := config.ValidateVolumeCatalog(merged); err != nil { + if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { return fmt.Errorf("runtime volume catalog: %w", err) } - m.publishRuntimeCatalog(m.runtimeVolumes) + m.catalog.entries, m.catalog.generation = entries, generation return nil } @@ -100,8 +159,8 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo if os.Geteuid() != 0 { return ErrVolumeCatalogReadOnly } - m.catalogWriteMu.Lock() - defer m.catalogWriteMu.Unlock() + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() if err := m.commitPendingCatalog(ctx); err != nil { return err } @@ -109,49 +168,32 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo if err != nil { return err } - m.mu.Lock() - if m.unknownVolumes > 0 { - m.mu.Unlock() - return ErrVolumeCatalogUnavailable + proposal, err := m.registrationProposal(name, normalized) + if err != nil || proposal == nil { + return err } - if current, ok := m.runtimeVolumes[name]; ok { - m.mu.Unlock() - if sameRegistration(current, normalized) { - return nil - } - return fmt.Errorf("%w: volume %q already has different semantics", ErrVolumeRegistrationConflict, name) + if st, statErr := m.statVolume(normalized.Path); statErr != nil || !st.Mode().IsRegular() { + return fmt.Errorf("%w: backing for volume %q must be an available regular file", ErrBadVolume, name) } - if m.volumeReferenced(name) { - m.mu.Unlock() - return fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) + return m.writeCatalog(ctx, name, proposal) +} + +func (m *Manager) registrationProposal(name string, spec types.VolumeRegistration) (*catalogWrite, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.catalog.admissionError(); err != nil { + return nil, err } - for existing, entry := range m.volumes { - if existing == name || filepath.Clean(entry.disk.Path) == normalized.Path { - m.mu.Unlock() - return fmt.Errorf("%w: volume %q duplicates a catalog name or path", ErrVolumeRegistrationConflict, name) + if entry := m.catalog.entries[name]; entry != nil { + if entry.runtime && sameRegistration(entry.registration(), spec) { + return nil, nil } - } - m.mu.Unlock() - if _, err := m.statVolume(normalized.Path); err != nil { - return fmt.Errorf("%w: backing for volume %q is unavailable", ErrBadVolume, name) - } - if err := ctx.Err(); err != nil { - return err - } - m.mu.Lock() - if m.unknownVolumes > 0 { - m.mu.Unlock() - return ErrVolumeCatalogUnavailable + return nil, fmt.Errorf("%w: volume %q already has different semantics or is static", ErrVolumeRegistrationConflict, name) } if m.volumeReferenced(name) { - m.mu.Unlock() - return fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) + return nil, fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) } - proposal := maps.Clone(m.runtimeVolumes) - proposal[name] = normalized - m.catalogPending = proposal - m.mu.Unlock() - return m.commitPendingCatalog(ctx) + return m.catalog.adding(name, spec) } // UnregisterVolume retires an unreferenced runtime binding without touching its backing. @@ -159,41 +201,41 @@ func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { if os.Geteuid() != 0 { return ErrVolumeCatalogReadOnly } - m.catalogWriteMu.Lock() - defer m.catalogWriteMu.Unlock() + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() if err := m.commitPendingCatalog(ctx); err != nil { return err } if !types.ValidVolumeName(name) { return fmt.Errorf("%w: invalid volume name", ErrBadVolume) } + proposal, err := m.removalProposal(name) + if err != nil || proposal == nil { + return err + } + return m.writeCatalog(ctx, name, proposal) +} + +func (m *Manager) removalProposal(name string) (*catalogWrite, error) { m.mu.Lock() - if m.unknownVolumes > 0 { - m.mu.Unlock() - return ErrVolumeCatalogUnavailable + defer m.mu.Unlock() + if err := m.catalog.admissionError(); err != nil { + return nil, err } - if _, runtime := m.runtimeVolumes[name]; !runtime { - _, exists := m.volumes[name] - referenced := m.volumeReferenced(name) - m.mu.Unlock() - if exists || referenced { - return fmt.Errorf("%w: volume %q is static or referenced", ErrVolumeRegistrationConflict, name) - } - return nil + entry := m.catalog.entries[name] + if (entry != nil && !entry.runtime) || m.volumeReferenced(name) { + return nil, fmt.Errorf("%w: volume %q is static or referenced", ErrVolumeRegistrationConflict, name) } - if m.volumeReferenced(name) { - m.mu.Unlock() - return fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) + if entry == nil { + return nil, nil } - proposal := maps.Clone(m.runtimeVolumes) - delete(proposal, name) - m.catalogPending = proposal - m.mu.Unlock() - return m.commitPendingCatalog(ctx) + entries := maps.Clone(m.catalog.entries) + delete(entries, name) + return &catalogWrite{entries: entries, generation: m.catalog.generation}, nil } func (m *Manager) volumeReferenced(name string) bool { - if holders := m.volumeAdmission[name]; holders.readers > 0 || holders.writers > 0 { + if holders := m.catalog.holders[name]; holders.readers > 0 || holders.writers > 0 { return true } for _, run := range m.volumeRuns { @@ -214,65 +256,92 @@ func (m *Manager) volumeReferenced(name string) bool { return false } +func (m *Manager) writeCatalog(ctx context.Context, name string, proposal *catalogWrite) error { + if err := validateRuntimeCatalogPath(m.dataDir); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogInvalid, err) + } + record := runtimeCatalog{Version: 1, Volumes: map[string]types.VolumeRegistration{}} + for key, entry := range proposal.entries { + if entry.runtime { + record.Volumes[key] = entry.registration() + } + } + raw, err := json.Marshal(record, json.Deterministic(true)) + if err != nil { + return fmt.Errorf("%w: %w", ErrBadVolume, err) + } + proposal.raw = raw + if err := m.acceptCatalog(ctx, name, proposal); err != nil { + return err + } + return m.persistCatalog(proposal) +} + +func (m *Manager) acceptCatalog(ctx context.Context, name string, proposal *catalogWrite) error { + m.mu.Lock() + defer m.mu.Unlock() + if err := ctx.Err(); err != nil { + return err + } + if err := m.catalog.admissionError(); err != nil { + return err + } + if m.volumeReferenced(name) { + return fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) + } + m.catalog.pending = proposal + return nil +} + func (m *Manager) commitPendingCatalog(ctx context.Context) error { m.mu.Lock() - proposal := m.catalogPending - unknown := m.unknownVolumes > 0 + proposal, unknown := m.catalog.pending, m.catalog.unknown > 0 m.mu.Unlock() - if unknown { - return ErrVolumeCatalogUnavailable - } if proposal == nil { return nil } + if unknown { + return ErrVolumeCatalogUnavailable + } if err := ctx.Err(); err != nil { return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) } - if err := m.replaceRuntimeCatalog(proposal); err != nil { + if err := validateRuntimeCatalogPath(m.dataDir); err != nil { return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) } - m.mu.Lock() - m.publishRuntimeCatalog(proposal) - m.catalogPending = nil - m.mu.Unlock() - return nil + return m.persistCatalog(proposal) } -func (m *Manager) publishRuntimeCatalog(proposal map[string]types.VolumeRegistration) { - for name := range m.runtimeVolumes { - if _, ok := proposal[name]; !ok { - delete(m.volumes, name) - } +func (m *Manager) persistCatalog(proposal *catalogWrite) error { + replace := m.catalog.replace + if replace == nil { + replace = func(path string, raw []byte) error { return utils.WriteFileSync(path, raw, 0o600) } } - for name, spec := range proposal { - if prior, ok := m.runtimeVolumes[name]; ok && sameRegistration(prior, spec) && m.volumes[name] != nil { - continue - } - m.volumes[name] = &catalogVolume{disk: engine.VolumeSpec{Name: name, Path: spec.Path, DirectIO: spec.DirectIO}, tenants: slices.Clone(spec.Tenants), writable: spec.Writable} + if err := replace(filepath.Join(m.dataDir, runtimeVolumesFile), proposal.raw); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) } - m.runtimeVolumes = proposal + m.mu.Lock() + m.catalog.entries, m.catalog.generation = proposal.entries, proposal.generation + m.catalog.pending = nil + m.mu.Unlock() + return nil } -func (m *Manager) replaceRuntimeCatalog(proposal map[string]types.VolumeRegistration) error { - if err := protectedCatalogPath(m.dataDir); err != nil { +func validateRuntimeCatalogPath(dir string) error { + if err := protectedCatalogPath(dir); err != nil { return err } - path := filepath.Join(m.dataDir, runtimeVolumesFile) - if st, err := os.Lstat(path); err == nil { - if !st.Mode().IsRegular() || st.Mode().Perm() != 0o600 || st.Sys().(*syscall.Stat_t).Uid != 0 { - return fmt.Errorf("runtime volume catalog must be a root-owned regular file with mode 0600") - } - } else if !errors.Is(err, os.ErrNotExist) { - return err + st, err := os.Lstat(filepath.Join(dir, runtimeVolumesFile)) + if errors.Is(err, os.ErrNotExist) { + return nil } - raw, err := json.Marshal(runtimeCatalog{Version: 1, Volumes: proposal}, json.Deterministic(true)) if err != nil { return err } - if m.catalogReplace != nil { - return m.catalogReplace(path, raw) + if !st.Mode().IsRegular() || st.Mode().Perm() != 0o600 || st.Sys().(*syscall.Stat_t).Uid != 0 { + return errors.New("runtime volume catalog must be a root-owned regular file with mode 0600") } - return utils.WriteFileSync(path, raw, 0o600) + return nil } func protectedCatalogPath(dir string) error { @@ -297,10 +366,10 @@ func protectedCatalogPath(dir string) error { } func (m *Manager) retryVolumeCatalog(ctx context.Context) { - if !m.catalogWriteMu.TryLock() { + if !m.catalog.writeMu.TryLock() { return } - defer m.catalogWriteMu.Unlock() + defer m.catalog.writeMu.Unlock() if err := m.commitPendingCatalog(ctx); err != nil { log.WithFunc("pool.retryVolumeCatalog").Error(ctx, err, "persist runtime volume catalog") } diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 67fd2e25..0c721752 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -1,8 +1,10 @@ package pool import ( + "context" "errors" "fmt" + "math" "os" "path/filepath" "slices" @@ -64,6 +66,97 @@ func TestRuntimeCatalogPersistsNormalizedSemantics(t *testing.T) { } } +func TestRuntimeCatalogInvalidDestinationDoesNotFenceStaticClaims(t *testing.T) { + requireCatalogRoot(t) + for _, kind := range []string{"symlink", "parent-owner", "parent-mode", "record-mode", "record-owner", "record-symlink"} { + t.Run(kind, func(t *testing.T) { + parent := t.TempDir() + dir := filepath.Join(parent, "state") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + static := liveCatalog(t) + m := newVolumeManagerAt(t, newFakeEngine(), dir, static) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("initial static claim: %v", err) + } + record := filepath.Join(dir, runtimeVolumesFile) + switch kind { + case "symlink": + link := filepath.Join(parent, "linked-state") + if err := os.Symlink(dir, link); err != nil { + t.Fatal(err) + } + m.dataDir = link + case "parent-owner": + if err := os.Chmod(parent, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chown(parent, 65534, 65534); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Chown(parent, 0, 0); err != nil { + t.Error(err) + } + }) + case "parent-mode": + if err := os.Chmod(parent, 0o777); err != nil { + t.Fatal(err) + } + default: + if err := os.WriteFile(record, []byte(`{"version":1,"volumes":{}}`), 0o600); err != nil { + t.Fatal(err) + } + switch kind { + case "record-mode": + if err := os.Chmod(record, 0o644); err != nil { + t.Fatal(err) + } + case "record-owner": + if err := os.Chown(record, 65534, 65534); err != nil { + t.Fatal(err) + } + case "record-symlink": + if err := os.Rename(record, record+".target"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(record+".target", record); err != nil { + t.Fatal(err) + } + } + } + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "unchanged")} + for range 2 { + err := m.RegisterVolume(t.Context(), "runtime", spec) + if !errors.Is(err, ErrVolumeCatalogInvalid) { + t.Errorf("invalid destination should be a permanent refusal: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Errorf("static claim after rejected registration: %v", err) + } + } + }) + } +} + +func TestRuntimeRegistrationRejectsNonregularBacking(t *testing.T) { + requireCatalogRoot(t) + for _, path := range []string{t.TempDir(), os.DevNull} { + t.Run(filepath.Base(path), func(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path, Writable: true}) + if !errors.Is(err, ErrBadVolume) { + t.Fatalf("nonregular backing registration: %v", err) + } + if names := m.VolumeNames(); len(names) != 0 { + t.Fatalf("rejected backing was published: %v", names) + } + }) + } +} + func TestRuntimeCatalogCollisionsAndRecoveryNames(t *testing.T) { requireCatalogRoot(t) eng := newFakeEngine() @@ -101,7 +194,7 @@ func TestRuntimeCatalogPostRenameFailureFencesAndConverges(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, nil) path := writeVolumeImage(t, "runtime", "bytes") - m.catalogReplace = func(path string, raw []byte) error { + m.catalog.replace = func(path string, raw []byte) error { if err := utils.WriteFileSync(path, raw, 0o600); err != nil { return err } @@ -121,11 +214,11 @@ func TestRuntimeCatalogPostRenameFailureFencesAndConverges(t *testing.T) { if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { t.Fatalf("volume-free claim blocked: %v", err) } - m.catalogReplace = nil + m.catalog.replace = nil if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { t.Fatalf("retry pending register: %v", err) } - m.catalogReplace = func(path string, raw []byte) error { + m.catalog.replace = func(path string, raw []byte) error { if err := utils.WriteFileSync(path, raw, 0o600); err != nil { return err } @@ -140,7 +233,7 @@ func TestRuntimeCatalogPostRenameFailureFencesAndConverges(t *testing.T) { if err := m.RegisterVolume(t.Context(), "replacement", types.VolumeRegistration{Path: path}); !errors.Is(err, ErrVolumeCatalogUnavailable) { t.Fatalf("pending unregister reuse: %v", err) } - m.catalogReplace = nil + m.catalog.replace = nil if err := m.RegisterVolume(t.Context(), "replacement", types.VolumeRegistration{Path: path}); err != nil { t.Fatalf("retry did not converge before new request: %v", err) } @@ -220,7 +313,7 @@ func TestRuntimeRegistrationNormalization(t *testing.T) { for _, tc := range []struct { name string spec types.VolumeRegistration - }{{"cocoon-reserved", input}, {"bad/name", input}, {"data", types.VolumeRegistration{Path: "relative"}}, {"data", types.VolumeRegistration{Path: "/data", DirectIO: "bad"}}} { + }{{"cocoon-reserved", input}, {"bad/name", input}, {"data", types.VolumeRegistration{Path: "relative"}}, {"data", types.VolumeRegistration{Path: "/data", DirectIO: "bad"}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{""}}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{"has space"}}}} { if _, err := normalizeRegistration(tc.name, tc.spec); !errors.Is(err, ErrBadVolume) { t.Fatalf("invalid registration %+v: %v", tc, err) } @@ -373,9 +466,9 @@ func TestRuntimeCatalogIOLeavesManagerAndTransitionUnlocked(t *testing.T) { var once sync.Once barrier := func() { once.Do(func() { close(entered) }); <-resume } if operation == "stat" { - m.volumeStat = func(path string) (os.FileInfo, error) { barrier(); return os.Stat(path) } + m.catalog.stat = func(path string) (os.FileInfo, error) { barrier(); return os.Stat(path) } } else { - m.catalogReplace = func(path string, raw []byte) error { + m.catalog.replace = func(path string, raw []byte) error { if err := utils.WriteFileSync(path, raw, 0o600); err != nil { return err } @@ -412,3 +505,174 @@ func TestRuntimeCatalogIOLeavesManagerAndTransitionUnlocked(t *testing.T) { }) } } + +func TestRuntimeCatalogCancellationBeforeAcceptanceDoesNotFenceClaims(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + m.catalog.stat = func(path string) (os.FileInfo, error) { cancel(); return os.Stat(path) } + m.catalog.replace = func(string, []byte) error { t.Fatal("canceled proposal attempted a write"); return nil } + err := m.RegisterVolume(ctx, "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}) + if !errors.Is(err, context.Canceled) || errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("cancellation before acceptance: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("canceled proposal left catalog: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("canceled registration fenced static claim: %v", err) + } +} + +func TestRuntimeCatalogCancellationAfterAcceptancePreservesWriteOutcome(t *testing.T) { + requireCatalogRoot(t) + for _, uncertain := range []bool{false, true} { + t.Run(fmt.Sprint(uncertain), func(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + m.catalog.replace = func(path string, raw []byte) error { + cancel() + if err := utils.WriteFileSync(path, raw, 0o600); err != nil { + return err + } + if uncertain { + return errors.New("directory sync result unavailable") + } + return nil + } + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + err := m.RegisterVolume(ctx, "runtime", spec) + if uncertain && !errors.Is(err, ErrVolumeCatalogUnavailable) || !uncertain && err != nil { + t.Fatalf("accepted write outcome: %v", err) + } + restarted := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil) + if !slices.Equal(restarted.VolumeNames(), []string{"runtime"}) { + t.Fatal("accepted replacement was not recoverable") + } + if uncertain { + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("uncertain replacement admitted volume: %v", err) + } + m.catalog.replace = nil + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("retry canceled request: %v", err) + } + } + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); err != nil { + t.Fatalf("completed replacement unavailable: %v", err) + } + }) + } +} + +func TestRuntimeCatalogPendingWriteSurvivesLaterPreflightRefusal(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + m.catalog.replace = func(path string, raw []byte) error { + if err := utils.WriteFileSync(path, raw, 0o600); err != nil { + return err + } + return errors.New("directory sync result unavailable") + } + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("initial uncertainty: %v", err) + } + m.catalog.replace = nil + record := filepath.Join(m.dataDir, runtimeVolumesFile) + if err := os.Chmod(record, 0o644); err != nil { + t.Fatal(err) + } + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("later invalid metadata cleared uncertainty: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unconfirmed replacement lost fence: %v", err) + } + if err := os.Chmod(record, 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := m.UnregisterVolume(ctx, "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("canceled retry cleared uncertainty: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if !slices.Contains(m.VolumeNames(), "runtime") { + t.Fatal("retry lost original accepted registration") + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("repair did not restore static admission: %v", err) + } +} + +func TestResolvedCatalogGenerationChangesOnlyForReusedName(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), nil) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) + if err != nil { + t.Fatal(err) + } + if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: writeVolumeImage(t, "other", "bytes")}); err != nil { + t.Fatal(err) + } + holds, err := m.admitVolumes("", resolved) + if err != nil { + t.Fatalf("unrelated registration invalidated resolution: %v", err) + } + m.unreserveVolumes(holds) + if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("same-file reuse admitted stale resolution: %v", err) + } +} + +func TestRuntimeCatalogGenerationExhaustionDoesNotWrite(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + m.catalog.generation = math.MaxUint64 + m.catalog.replace = func(string, []byte) error { t.Fatal("exhausted generation attempted write"); return nil } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("generation exhaustion: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("exhaustion fenced existing registration: %v", err) + } +} + +func TestRuntimeCatalogIgnoresAbandonedTemporaryFiles(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), nil) + path := writeVolumeImage(t, "runtime", "bytes") + raw := fmt.Sprintf(`{"version":1,"volumes":{"abandoned":{"path":%q}}}`, path) + tmp := filepath.Join(m.dataDir, runtimeVolumesFile+".interrupted.tmp") + if err := os.WriteFile(tmp, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("temporary record promoted without canonical record: %v", got) + } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatal(err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("temporary record overrode canonical record: %v", got) + } + if got, err := os.ReadFile(tmp); err != nil || string(got) != raw { + t.Fatalf("abandoned evidence changed: %q %v", got, err) + } +} diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index 6a476302..58f1b015 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -12,6 +12,7 @@ import ( "testing/synctest" "time" + "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -367,3 +368,118 @@ func TestCanceledRemovalDispatchKeepsUnknownFenceOwned(t *testing.T) { } }) } + +func TestWarmAttachBeforeFinalizeNeedsUnknownFenceAfterRestart(t *testing.T) { + base := newFakeEngine() + name := "sbx-warm-unfinalized" + base.vms[name] = "/vsock/warm" + base.diskAttachErrFor = "broken" + base.removeErrFor = name + rendezvousAttaches(base, 2) + catalog := []config.VolumeSpec{ + {Name: "data", Path: writeVolumeImage(t, "warm-data", "data"), Writable: true}, + {Name: "broken", Path: writeVolumeImage(t, "warm-broken", "broken")}, + } + dir := t.TempDir() + m := newVolumePoolManager(t, base, dir, catalog) + m.eng = &emptyWarmInventoryEngine{fakeEngine: base} + warm := &types.Sandbox{VMName: name, Key: testKey, VsockSocket: "/vsock/warm"} + m.pools[testKey].warm = append(m.pools[testKey].warm, warm) + _, err := m.ClaimWarm(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{ + {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}, + {Name: "broken", AttachOnly: true}, + }}) + if err == nil { + t.Fatal("expected attach failure before finalization") + } + if warm.ID != "" { + t.Fatal("claim unexpectedly finalized") + } + base.mu.Lock() + attached := slices.Clone(base.disks[name]) + base.attachRendezvous = nil + base.mu.Unlock() + if !slices.ContainsFunc(attached, func(d types.VMDisk) bool { return d.Name == "data" && !d.ReadOnly }) { + t.Fatalf("warm VM has no attached writer: %+v", attached) + } + claims, err := newClaimStore(dir, false).load() + if err != nil || len(claims) != 0 { + t.Fatalf("pre-finalize journal: %+v, %v", claims, err) + } + restarted := newVolumeManagerAt(t, base, dir, catalog) + restarted.eng = &emptyWarmInventoryEngine{fakeEngine: base} + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("ambiguous warm survivor admitted competing writer: %v", err) + } + base.removeErrFor = "" + restarted.retryRemovals(t.Context()).Wait() + if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("confirmed removal failed to clear fence: %v", err) + } +} + +type emptyWarmInventoryEngine struct{ *fakeEngine } + +func (e *emptyWarmInventoryEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + rec, present, err := e.fakeEngine.Inspect(ctx, name) + rec.AttachedDevices.Disks = nil + return rec, present, err +} + +func (e *emptyWarmInventoryEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { + if err := e.fakeEngine.DiskAttach(ctx, name, spec); err != nil { + return err + } + e.mu.Lock() + if e.disks == nil { + e.disks = map[string][]types.VMDisk{} + } + e.disks[name] = append(e.disks[name], types.VMDisk{Name: spec.Name, ReadOnly: !spec.RW}) + e.mu.Unlock() + return nil +} + +func TestArchivedReleaseWithUnvalidatedVolumeRunHasNoCleanupHolds(t *testing.T) { + base := newFakeEngine() + m := newTestManager(t, base, archivePool(3600)) + sb := mustClaim(t, m, testKey) + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if err := m.archive(t.Context(), sb); err != nil { + t.Fatal(err) + } + if sb.VMName != "" { + t.Fatalf("archived VM name = %q", sb.VMName) + } + ctx, run, err := m.beginVolumeRun(t.Context(), sb) + if err != nil { + t.Fatal(err) + } + if err := m.checkVolumeRun(ctx, sb, run); !errors.Is(err, ErrArchived) { + t.Fatalf("archived mutation validation: %v", err) + } + td := m.quiesceVolumes(t.Context(), sb) + if td.done == nil || len(td.holds) != 0 || len(td.clears) != 0 || td.unknown { + t.Fatalf("archived teardown has cleanup obligations: %+v", td) + } + if m.finishVolumeTeardown(t.Context(), td) { + t.Fatal("open volume run unexpectedly completed") + } + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + m.finishVolumeRun(sb, run) + if len(m.pendingRemovals) != 0 { + t.Fatalf("empty teardown enqueued removal: %+v", m.pendingRemovals) + } + if slices.Contains(base.removedNames(), "") { + t.Fatal("release removed an empty VM name") + } + if len(m.store.recoveryVolumes()) != 0 { + t.Fatal("release retained archived journal references") + } +} diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 7c002ecc..659ec1a5 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -22,7 +22,7 @@ func TestSettledVolumesDoNotCallDevicesOrResolveCatalog(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } - delete(m.volumes, "data") + delete(m.catalog.entries, "data") before := len(eng.volumeSpecs) if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "other"}}); err != nil { t.Fatal(err) @@ -57,9 +57,9 @@ func TestRemovedTenantRetainsAndRepairsHeldVolumes(t *testing.T) { t.Fatal("expected detaching state") } eng.detachVolumeErr = nil - entry := m.volumes["data"] + entry := m.catalog.entries["data"] entry.tenants = []string{"other-tenant"} - m.volumes["data"] = entry + m.catalog.entries["data"] = entry if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { t.Fatalf("held repair checked new authorization: %v", err) } diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go index e5e573c2..f4187b58 100644 --- a/sandboxd/pool/volume_retirement_test.go +++ b/sandboxd/pool/volume_retirement_test.go @@ -258,9 +258,9 @@ func TestRecoveryOnlyRetryPreservesHeldRepairAuthorization(t *testing.T) { t.Fatal(err) } m.mu.Lock() - entry := *m.volumes["data"] + entry := *m.catalog.entries["data"] entry.tenants = []string{"different-tenant"} - m.volumes["data"] = &entry + m.catalog.entries["data"] = &entry m.mu.Unlock() if err := os.Remove(tmp); err != nil { t.Fatal(err) @@ -275,3 +275,44 @@ func TestRecoveryOnlyRetryPreservesHeldRepairAuthorization(t *testing.T) { t.Fatalf("repair exemption admitted unrelated addition: %v", err) } } + +func TestClaimSnapshotAcknowledgesCapturedRowsBeforeNewerReplacement(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data"}}} + old := s.set(sb) + entered, resume := make(chan struct{}), make(chan struct{}) + s.replaceFile = func(raw []byte) error { close(entered); <-resume; return s.replace(raw) } + done := make(chan error, 1) + go func() { done <- s.commit(old) }() + <-entered + newer := s.del(sb.ID) + close(resume) + if err := <-done; err != nil { + t.Fatal(err) + } + if !s.pending(newer) { + t.Fatal("captured snapshot acknowledged newer deletion") + } + if refs := s.recoveryVolumes()[sb.ID]; len(refs) != 1 || refs[0].Name != "data" { + t.Fatalf("captured row not acknowledged: %+v", refs) + } + persisted, err := newClaimStore(filepath.Dir(s.path), false).load() + if err != nil || persisted[sb.ID] == nil { + t.Fatalf("captured disk snapshot: %+v, %v", persisted, err) + } + s.replaceFile = nil + if err = s.commit(newer); err != nil { + t.Fatal(err) + } + if len(s.recoveryVolumes()) != 0 { + t.Fatal("empty replacement retained old reference") + } + s.replaceFile = func([]byte) error { t.Fatal("old token attempted another replacement"); return nil } + if err = s.commit(old); err != nil { + t.Fatal(err) + } + persisted, err = newClaimStore(filepath.Dir(s.path), false).load() + if err != nil || len(persisted) != 0 { + t.Fatalf("old token resurrected snapshot: %+v, %v", persisted, err) + } +} diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index a81ff1bb..033bc8c3 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -229,7 +229,7 @@ func TestReserveVolumesExcludesReaderUnderWriter(t *testing.T) { if err := m.reserveVolumes([]types.Volume{{Name: "scratch", Mount: "/volumes/scratch"}}); !errors.Is(err, ErrVolumeBusy) { t.Errorf("read-only reservation under a writer: %v, want ErrVolumeBusy", err) } - if holders := m.volumeAdmission["scratch"]; holders != (volumeHolders{writers: 1}) { + if holders := m.catalog.holders["scratch"]; holders != (volumeHolders{writers: 1}) { t.Errorf("registry after the refusal=%+v, want the writer alone", holders) } } @@ -580,5 +580,5 @@ func newVolumeManagerAt(t *testing.T, eng *fakeEngine, dataDir string, volumes [ func volumeHoldersOf(m *Manager, name string) volumeHolders { m.mu.Lock() defer m.mu.Unlock() - return m.volumeAdmission[name] + return m.catalog.holders[name] } diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 4ad768a6..f9f7ad9a 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -49,6 +49,7 @@ var poolErrHTTP = []struct { {pool.ErrBadVolume, http.StatusBadRequest, ""}, {pool.ErrVolumeRegistrationConflict, http.StatusConflict, pool.ErrVolumeRegistrationConflict.Error()}, {pool.ErrVolumeCatalogReadOnly, http.StatusConflict, pool.ErrVolumeCatalogReadOnly.Error()}, + {pool.ErrVolumeCatalogInvalid, http.StatusConflict, pool.ErrVolumeCatalogInvalid.Error()}, {pool.ErrVolumeCatalogUnavailable, http.StatusServiceUnavailable, "volume catalog unavailable; retry"}, {pool.ErrBadName, http.StatusBadRequest, ""}, {pool.ErrBadCount, http.StatusBadRequest, ""}, diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go index 9916ce8c..3dc27e4f 100644 --- a/sandboxd/server/volume_registration_test.go +++ b/sandboxd/server/volume_registration_test.go @@ -62,7 +62,7 @@ func TestVolumeRegistrationRoutesRequireRoot(t *testing.T) { func TestVolumeRegistrationRequiresConfiguredTokenBeforeReadingBody(t *testing.T) { path := filepath.Join(t.TempDir(), "data.img") - if err := os.WriteFile(path, []byte("existing backing"), 0600); err != nil { + if err := os.WriteFile(path, []byte("existing backing"), 0o600); err != nil { t.Fatal(err) } for _, method := range []string{http.MethodPut, http.MethodDelete} { @@ -213,6 +213,7 @@ func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { {"invalid registration", fmt.Errorf("%w: invalid directio", pool.ErrBadVolume), http.StatusBadRequest}, {"binding conflict", fmt.Errorf("%w: /private/data.img", pool.ErrVolumeRegistrationConflict), http.StatusConflict}, {"OS root required", pool.ErrVolumeCatalogReadOnly, http.StatusConflict}, + {"invalid catalog destination", fmt.Errorf("%w: /private/runtime-volumes.json", pool.ErrVolumeCatalogInvalid), http.StatusConflict}, {"catalog unavailable", fmt.Errorf("%w: sync /private/runtime-volumes.json", pool.ErrVolumeCatalogUnavailable), http.StatusServiceUnavailable}, {"uncertain cancellation", fmt.Errorf("%w: %w", pool.ErrVolumeCatalogUnavailable, context.Canceled), http.StatusServiceUnavailable}, {"uncertain timeout", fmt.Errorf("%w: %w", pool.ErrVolumeCatalogUnavailable, context.DeadlineExceeded), http.StatusServiceUnavailable}, From 66e1dfd3c74b4eaaa272a1773da52352be4dc221 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 01:25:33 +0000 Subject: [PATCH 15/72] catalog: clarify registration guarantees and harden CI --- .github/workflows/sandboxd.yml | 12 ++++++++---- docs/deploy.md | 6 ++++++ docs/sandboxd-api.md | 13 +++++++++---- docs/security.md | 7 +++++-- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/sandboxd.yml b/.github/workflows/sandboxd.yml index c96b1d73..61c312b6 100644 --- a/.github/workflows/sandboxd.yml +++ b/.github/workflows/sandboxd.yml @@ -67,14 +67,16 @@ jobs: - name: Test run: make go-test - - name: Test runtime catalog as root + - name: Test pool as root working-directory: sandboxd shell: bash run: | GOWORK=off go test -race -c -o "$RUNNER_TEMP/catalog.test" ./pool - sudo "$RUNNER_TEMP/catalog.test" -test.count=1 -test.v -test.run 'Test(RuntimeCatalog|RuntimeRegistration|ResolvedCatalog|IncompleteStaleInventory|RemovedVMKeepsVolume|DispatchedCleanup|UnknownStaleVolumes)' | tee "$RUNNER_TEMP/catalog-test.log" + sudo setpriv --bounding-set=-dac_override,-dac_read_search "$RUNNER_TEMP/catalog.test" -test.count=1 -test.v -test.skip '^Test(PromoteFailsClosedOnMetaError|NodesOfACellShareThePoolSet|AMetaStoreNodeWithoutACellKeepsItsOwnPools|TwoNodesOnOneMetaStoreShareTheTenantSet)$' | tee "$RUNNER_TEMP/catalog-test.log" grep -q '^--- PASS: TestRuntimeCatalogPersistsNormalizedSemantics ' "$RUNNER_TEMP/catalog-test.log" - ! grep -- '--- SKIP' "$RUNNER_TEMP/catalog-test.log" + if grep -- '--- SKIP:' "$RUNNER_TEMP/catalog-test.log"; then + exit 1 + fi - name: Test the PostgreSQL meta store working-directory: sandboxd @@ -84,7 +86,9 @@ jobs: run: | GOWORK=off go test -race -count=1 -v ./tenants/... ./poolset/... | tee "$RUNNER_TEMP/pg-test.log" GOWORK=off go test -race -count=1 -v -run 'TestTwoNodesOnOneMetaStore|TestNodesOfACell|TestAMetaStoreNodeWithoutACell' ./pool/ | tee -a "$RUNNER_TEMP/pg-test.log" - ! grep -- '--- SKIP' "$RUNNER_TEMP/pg-test.log" + if grep -- '--- SKIP:' "$RUNNER_TEMP/pg-test.log"; then + exit 1 + fi - name: Set up Python uses: actions/setup-python@v7.0.0 diff --git a/docs/deploy.md b/docs/deploy.md index f8020f0e..b72b59cc 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -247,6 +247,12 @@ the pool had `reader`, and its reads wake the claim as without the feature. ### Dataset volumes +Volume `tenants` entries in static configuration must match the tenant name +syntax used by the tenant API. Configurations containing invalid names that +previously only produced warnings now fail validation at startup. Remove or +correct those entries before upgrading. Valid names for tenants that have not +yet been created remain accepted. + Running claims can change mounted read-only catalog entries through the [GET, PUT, and PATCH volume API](sandboxd-api.md#get-v1sandboxesidvolumes) and both SDKs. The image must already exist on the owner node. Live changes diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 44243500..3cb3c0c9 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -272,8 +272,10 @@ JSON return 400. - `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. - `path` uses the same absolute-path validation as static volume config. - Registration also requires an existing regular file. A symlink to a regular - file is allowed; directories and device files return 400. + Creating a registration requires an existing regular file. A symlink to a + regular file is allowed; directories and device files return 400. This check + runs only when creating the registration, not on idempotent retries or + attachment. Operators must keep the registered path and target stable. - `directio` is `on`, `off`, or `auto`, and defaults to `off`. - `writable` defaults to `false` and controls writable claim admission. - `tenants` limits use to the named tenants. Omitted or empty permits all tenants. @@ -284,10 +286,13 @@ Success is 204 with an empty body. Repeating the same name, cleaned path, and normalized registration fields succeeds. A different binding or different registration fields return 409. Static and runtime registrations cannot share a name or a cleaned backing path. Invalid registration fields return 400. -Invalid catalog directory or file ownership, permissions, or type returns 409 +Violations of the catalog ownership, mode, directory, or file-type policy return 409 without `Retry-After`, before accepting a new write. This refusal does not block claims using the existing catalog. The storage configuration must be repaired -before that mutation can succeed. +before that mutation can succeed. These checks cover metadata ownership, +permissions, and path types, not every storage failure. Read-only filesystems, +ENOSPC, immutable attributes, or access errors during the actual write still +enter persistence recovery and can block volume admission until storage is repaired. An uncertain write returns 503 with `Retry-After`. The requested registration may still take effect through background retry or a later PUT or DELETE, diff --git a/docs/security.md b/docs/security.md index 5808c3ad..8c4a9acd 100644 --- a/docs/security.md +++ b/docs/security.md @@ -106,8 +106,11 @@ exposes host image paths or access lists. Runtime registration and unregistration require a configured root `api_token`; these endpoints return 403 when the token is unset. The daemon must also run as OS root to change the root-owned registration file. Runtime registration -accepts existing regular files, including symlinks to regular files. Operators -remain responsible for selecting backing paths and coordinating aliases. +checks that the backing resolves to a regular file when creating a registration. +Symlinks to regular files are allowed. An idempotent registration retry does not +repeat this check, and attachment does not recheck the file type or pin the +backing identity. Operators must prevent backing paths or symlink targets from +being replaced while registered and coordinate aliases. Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path From b76a92d55bfc7e6683578b05ceed2dd7fae04707 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 02:07:48 +0000 Subject: [PATCH 16/72] volumes: support writable and attach-only live changes --- docs/live-volumes.md | 39 ++- docs/sandboxd-api.md | 51 +++- docs/security.md | 7 + e2e/fakeengine_test.go | 6 +- e2e/volume_mutation_test.go | 2 +- sandboxd/engine/volume_live.go | 39 ++- sandboxd/engine/volume_live_test.go | 57 +++- sandboxd/pool/claims.go | 8 + sandboxd/pool/claims_test.go | 29 ++ sandboxd/pool/pool.go | 2 +- sandboxd/pool/pool_test.go | 14 +- sandboxd/pool/volume.go | 4 +- sandboxd/pool/volume_attach_test.go | 8 +- sandboxd/pool/volume_modes_test.go | 387 ++++++++++++++++++++++++ sandboxd/pool/volume_mutation.go | 61 ++-- sandboxd/pool/volume_mutation_test.go | 8 +- sandboxd/server/volume_mutation_test.go | 42 +++ sandboxd/types/types.go | 14 +- sandboxd/types/volume_test.go | 25 +- 19 files changed, 700 insertions(+), 103 deletions(-) create mode 100644 sandboxd/pool/volume_modes_test.go diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 19dac036..f787ecda 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -16,8 +16,8 @@ external controller owns quiescence and unmount before detach, a reservation through sleep, and access gating until wake, attach, and mount complete. That gate also covers business requests that wake the sandbox. Wake establishes only silkd readiness. The API does not include a controller implementation or -verify that composed workflow. Live changes remain limited to mounted -read-only entries. +verify that composed workflow. Live changes support mounted and attach-only +entries in read-only or writable mode. Pause application reads before replacement. Close files, mmap mappings, cwd, and bind mount references on the old disk. Replacement unmounts the old disk @@ -68,9 +68,9 @@ with client.new("rt:24.04", ttl_seconds=900) as sb: A successful run reads `v1`, then `v2`, and preserves every identity assertion. The repeated full target verifies that a settled mount can be retained. -This example starts without other volumes. If your sandbox has writable or -attach-only volumes, retain them in a full PUT or use PATCH to replace only -the dataset names. `None`, a nil Go slice, and a missing `volumes` field do not +This example starts without other volumes. A full PUT replaces the entire +desired set, including writable and attach-only entries. Retain other entries +in that list, or use PATCH to replace only the dataset names. `None`, a nil Go slice, and a missing `volumes` field do not mean an empty target. Go uses the same owning handle and client API token. @@ -101,3 +101,32 @@ admission holds. Detailed errors survive only within one daemon lifetime. If the VMM exits, a sandbox holding volumes is release-only. Release it and create a new sandbox; neither PUT with an empty target nor daemon restart restores the original processes. + +## Writable and attach-only entries + +Use `mode: "rw"` for a catalog entry that permits writing. Set per-entry `attach_only: true` when the caller mounts the device itself. Both choices can be +mixed with mounted read-only entries in PUT, PATCH, or a new claim: + +```json +{"volumes":[{"name":"dataset"},{"name":"scratch","mode":"rw"},{"name":"raw-data","attach_only":true},{"name":"raw-work","mode":"rw","attach_only":true}]} +``` + +Omitting a mount still selects `/volumes/` for a new mounted entry. An +attach-only entry cannot supply a nonempty mount. Responses preserve the flag +through GET, PUT retries, and both SDKs. The existing Go claim option +`WithVolumesAttachOnly()` and Python `mount=False` remain claim-wide aliases. + +Before removal, stop application access and close references. For attach-only +devices, unmount the caller-managed filesystem first. sandboxd checks its guest +mount namespace, but cannot prove that raw block users, partitions, or other +mount namespaces are idle. Keep the control-plane reservation across any sleep +and gate access until wake, attach, and mount complete. + +For mounted writable entries, sandboxd clears its dirty marker only after +observing a normal unmount of the expected writable root mount and confirming +host detach. External pre-unmount, an absent device, or a lost result leaves the +marker conservatively in place. Detach can succeed while a subsequent read-only +attach still requires recovery. Attach-only writers have no daemon-managed +marker and remain the caller's filesystem responsibility. See the [API +contract](sandboxd-api.md#patch-v1sandboxesidvolumes) for failure and retry +semantics. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 3cb3c0c9..fd227fad 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -135,12 +135,15 @@ an existing populated guest directory for the claim's life. ### Attach-only volumes -`"volumes_attach_only": true` stops after the attach. The device appears in -the guest, nothing is mounted, and the echoed entries carry no `mount` key: +Set `"attach_only": true` on an individual volume to stop after attachment. +It can share a claim with mounted entries. The existing claim-wide +`"volumes_attach_only": true` applies that choice to every entry. A nonempty +`mount` is invalid with either form. The device appears in the guest without a +mount. Echoed entries carry `attach_only: true` and no `mount` key: ```json {"id": "sb_…", "token": "…", - "volumes": [{"name": "imagenet"}, {"name": "scratch-db", "mode": "rw"}]} + "volumes": [{"name": "imagenet", "attach_only": true}, {"name": "scratch-db", "mode": "rw", "attach_only": true}]} ``` Find each device by its virtio serial, which is the catalog name — poll this, @@ -301,7 +304,7 @@ acceptance, so it does not promise that the request will take effect. After repairing storage failures, retry the exact request to establish completion. Cancellation is not a rollback once a write has been accepted. GET retains its existing projection without host paths or tenant access lists. -Registration does not expand live mutation beyond mounted read-only volumes. +Registered entries support the live volume modes described below. Only registration metadata is persisted locally in `data_dir/runtime-volumes.json`, which loads before claim recovery. The record @@ -390,19 +393,24 @@ Replaces the full desired set and converges synchronously. Auth matches GET. The owner tenant's catalog access list applies to newly admitted names, including for root. Retaining, repairing, or removing held names does not reauthorize them. A new image must exist on the owning node. The request must include a non-null `volumes` -array. An explicit empty array removes all mutable entries. +array. An explicit empty array removes all entries. ```json {"volumes": [{"name": "imagenet", "mount": "/datasets/training"}]} ``` -Only mounted read-only catalog entries can change live (`mode` omitted or -`ro`). A PUT must retain every existing writable or attach-only entry unchanged. +Mounted read-only, mounted writable, attach-only read-only, and attach-only +writable entries can be added or removed live. `mode` defaults to read-only; +`rw` requires a writable catalog entry and excludes other readers and writers. +Set `attach_only: true` on an entry to attach its device without mounting it. +A nonempty `mount` with that flag is invalid. The legacy claim-wide +`volumes_attach_only` flag still applies to every entry in a new claim. The claim-time name, mount, and eight-volume limits apply to the desired set. -An omitted mount defaults to `/volumes/`, except that a held attach-only -entry is retained by sending it as the claim returned it, without `mount`. To -change an existing name's -mount or mode, first remove that name and wait for successful convergence. +A new entry with omitted `mount` defaults to `/volumes/`. Responses emit +`attach_only: true` for device-only entries, so GET's desired list can be sent +back as a PUT. Older clients can still retain a held mountless entry by +omitting both fields. To change an existing name's mount, mode, or attach-only +shape, first remove that name and wait for successful convergence. An identical target can be retried. Claims with `on_expire: archive` must change that policy before adding a volume. @@ -426,13 +434,30 @@ unchanged. An empty patch retries the current desired set. ``` Adding an identical entry or removing an absent valid name is harmless. -Duplicate names, a name in both arrays, or changes to writable or attach-only -entries return 400. Removal refuses busy or foreign mounts and uses neither +Duplicate names, a name in both arrays, or changes to an existing held entry's +mount, mode, or attach-only shape return 400. Removal refuses busy or foreign mounts and uses neither lazy nor forced unmount. A foreign mount, another bind mount, or a completed ordinary unmount with nonzero exit returns 409 with the guest mount path and a safe per-volume reason. Guest transport failures remain server errors; timeouts return 504 and cancellation returns 408. +For attach-only entries, the caller must stop writes and unmount before detach. +The daemon refuses mounts it can observe for the device; it does not unmount a +caller-managed filesystem. That probe does not establish that raw block users, +partitions, or other mount namespaces are idle. The controller owns those +conditions. + +Mounted writable entries receive a dirty marker before attachment. Live removal +clears it only after sandboxd observes the expected writable root mount, +completes a normal unmount, confirms no remaining device mounts, and receives +successful host detach. Cleanup finishes before writer admission is released. +An already absent device, an externally unmounted filesystem, or uncertain +host removal supplies no clean-unmount evidence. Detach can complete while the +marker remains, and a later read-only request can return 409 requiring recovery. +A marker-file removal failure is logged and retains the conservative marker. +Attach-only writers have no daemon-managed dirty marker; their caller owns +filesystem consistency. + ### Partial failure and recovery PUT and PATCH return the GET response shape on success. A convergence error diff --git a/docs/security.md b/docs/security.md index 8c4a9acd..ee2787c0 100644 --- a/docs/security.md +++ b/docs/security.md @@ -139,6 +139,13 @@ to every other tenant's next read. A dataset that genuinely needs multiple writers needs an out-of-band process for who writes when, which the catalog ACL does not provide. +Live writable and attach-only changes enforce the same catalog ACL and local +reader and writer admission as initial claims. Attach-only callers own quiescence, +unmount, and filesystem consistency. Guest mountinfo checks do not prove the +absence of raw block users, partition users, or users in other mount namespaces. +A mounted writable dirty marker is cleared only with observed normal unmount +and confirmed host detach; missing evidence leaves it in place. + ## Known limitations Facts to plan around, stated so the boundary is honest: diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 9bbc9113..d970831c 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -215,14 +215,14 @@ func (f *fakeEngine) SyncGuest(_ context.Context, _ string) error { return nil } -func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) error { +func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) (engine.VolumeDetachResult, error) { if err := f.UnmountVolume(ctx, sock, mount); err != nil { - return err + return engine.VolumeDetachResult{}, err } f.mu.Lock() defer f.mu.Unlock() f.volumeOps = append(f.volumeOps, "detach:"+name) - return nil + return engine.VolumeDetachResult{}, nil } func (f *fakeEngine) volumeOpsLog() []string { diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index 5fcedef8..ccd6d01d 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -52,7 +52,7 @@ func TestRejectedVolumeTargetExplainsRefusalWithoutChangingState(t *testing.T) { t.Fatal(checkErr) } _, err = sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}, {Name: "dataset"}}) - if refusal, ok := errors.AsType[*sandbox.VolumeError](err); !ok || refusal.Status != http.StatusBadRequest || !strings.Contains(refusal.Message, "duplicate volume") { + if refusal, ok := errors.AsType[*sandbox.VolumeError](err); !ok || refusal.Status != http.StatusBadRequest || !strings.Contains(refusal.Message, "duplicates name") { t.Errorf("invalid target explanation: %v", err) } view, err := sb.GetVolumes(t.Context()) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index e1033a84..ec61e655 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -62,39 +62,56 @@ func (e *Engine) MountLiveVolume(ctx context.Context, vsockSocket, name, mount s return e.mountDevice(ctx, vsockSocket, device, name, mount, rw) } +// VolumeDetachResult reports clean-unmount evidence from a confirmed host detach. +type VolumeDetachResult struct { + NormalUnmount bool +} + // DetachVolume unmounts a guest volume before confirming host removal. -func (e *Engine) DetachVolume(ctx context.Context, vmName, vsockSocket, name, mount string) error { +func (e *Engine) DetachVolume(ctx context.Context, vmName, vsockSocket, name, mount string) (VolumeDetachResult, error) { ctx, cancel := context.WithTimeout(ctx, VolumeMutationTimeout) defer cancel() device, found, err := e.findVolumeDevice(ctx, vsockSocket, name) if err != nil { - return err + return VolumeDetachResult{}, err } + normal := false if found { - if err = e.unmountLiveVolume(ctx, vsockSocket, device, mount); err != nil { - return err + if normal, err = e.unmountLiveVolume(ctx, vsockSocket, device, mount); err != nil { + return VolumeDetachResult{}, err } } - _, err = e.run(ctx, "vm", "disk", "detach", vmName, argName, name) - return err + if _, err = e.run(ctx, "vm", "disk", "detach", vmName, argName, name); err != nil { + return VolumeDetachResult{}, err + } + return VolumeDetachResult{NormalUnmount: normal}, nil } -func (e *Engine) unmountLiveVolume(ctx context.Context, vsockSocket, device, mount string) error { +func (e *Engine) unmountLiveVolume(ctx context.Context, vsockSocket, device, mount string) (bool, error) { + if mount == "" { + return false, e.refuseOtherMounts(ctx, vsockSocket, device) + } devNum, mounts, err := e.volumeMounts(ctx, vsockSocket, device) if err != nil { - return err + return false, err } target := e.resolveMount(ctx, vsockSocket, mount, mounts) atTarget := slices.DeleteFunc(mounts, func(m volumeMount) bool { return m.path != target }) + normal := false if len(atTarget) == 1 && atTarget[0].devNum == devNum && atTarget[0].root == "/" { if err = e.UnmountVolume(ctx, vsockSocket, mount); err != nil { if _, ok := errors.AsType[*guestExitError](err); ok { - return &VolumeMountConflict{Message: fmt.Sprintf("cannot unmount %q; close guest references and retry", mount)} + return false, &VolumeMountConflict{Message: fmt.Sprintf("cannot unmount %q; close guest references and retry", mount)} } - return err + return false, err } + mode, _, _ := strings.Cut(atTarget[0].options, ",") + normal = mode == "rw" + } + if err = e.refuseOtherMounts(ctx, vsockSocket, device); err != nil { + return false, err } - return e.refuseOtherMounts(ctx, vsockSocket, device) + return normal, nil } func (e *Engine) volumeMounts(ctx context.Context, vsockSocket, device string) (string, []volumeMount, error) { diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index a731f2e1..a3aa2f74 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -54,7 +54,7 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { t.Fatal(err) } } - if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, tt.mount); err != nil { + if _, err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, tt.mount); err != nil { t.Fatal(err) } fake.mu.Lock() @@ -191,7 +191,7 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n3 2 8:32 /sub /data ro - ext4 /dev/vdc ro\n") } fake.mu.Unlock() - err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") + _, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") if err == nil { t.Fatal("detach must fail closed") } @@ -222,7 +222,7 @@ func TestLiveDetachMissingGuestStillChecksHost(t *testing.T) { sock := sockPath(t) serveFakeSilkd(t, sock) spec := liveDiskCLI(t, hostErr) - err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") + _, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") wantErr := hostErr != "" if (err != nil) != wantErr { t.Fatalf("got %v, want error=%v", err, wantErr) @@ -240,7 +240,7 @@ func TestLiveDetachIgnoresAnotherDeviceAtThePath(t *testing.T) { fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:48 / /data ro - ext4 /dev/vdd ro\n") fake.mu.Unlock() - if err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err != nil { + if _, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err != nil { t.Fatalf("detach of an unmounted device blocked by another device's mount: %v", err) } fake.mu.Lock() @@ -282,3 +282,52 @@ exit 2 func guestChanges(calls [][]string) [][]string { return slices.DeleteFunc(slices.Clone(calls), func(argv []string) bool { return argv[0] == "realpath" }) } + +func TestLiveDetachCleanEvidenceRequiresWritableUnmountAndHostSuccess(t *testing.T) { + for _, mode := range []string{"normal", "readonly", "pre-unmounted", "missing device", "host failure", "busy", "residual mount", "attach-only mounted"} { + t.Run(mode, func(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + if mode != "missing device" { + configureVolumeDevices(fake) + } + hostErr := "" + if mode == "host failure" { + hostErr = "host removal failed after unmount" + } + spec := liveDiskCLI(t, hostErr) + mount := "/data" + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw - ext4 /dev/vdc rw\n") + switch mode { + case "pre-unmounted", "missing device": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + case "readonly": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n") + case "busy": + fake.execCode = 32 + case "attach-only mounted": + mount = "" + } + fake.execHook = func(argv []string) { + if argv[0] == "umount" && mode != "residual mount" && mode != "busy" { + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + } + } + fake.mu.Unlock() + result, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, mount) + wantErr := slices.Contains([]string{"host failure", "busy", "residual mount", "attach-only mounted"}, mode) + if (err != nil) != wantErr || result.NormalUnmount != (mode == "normal") { + t.Fatalf("mode=%s result=%+v err=%v", mode, result, err) + } + if mode == "attach-only mounted" { + fake.mu.Lock() + defer fake.mu.Unlock() + if len(guestChanges(fake.execCalls)) != 0 { + t.Fatal("attach-only detach unmounted a caller-managed filesystem") + } + } + }) + } +} diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index c4ae1345..768e63cb 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -111,6 +111,14 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { return nil, fmt.Errorf("parse claims: %w", err) } for _, sb := range claims { + for _, volumes := range [][]types.Volume{sb.Volumes, sb.VolumesWant} { + for i := range volumes { + if volumes[i].AttachOnly && volumes[i].Mount != "" { + return nil, fmt.Errorf("claim %q has a contradictory volume mount", sb.ID) + } + volumes[i].AttachOnly = volumes[i].Mount == "" + } + } if sb.VolumesWant == nil { sb.VolumesWant = sb.Volumes } diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index 0539a537..f40870b0 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -231,3 +231,32 @@ func TestClaimStoreConcurrentCommits(t *testing.T) { t.Fatalf("after concurrent commits: %+v, %v", got, err) } } + +func TestClaimLoadRestoresAttachOnlyBeforeRecoveryReferences(t *testing.T) { + dir := t.TempDir() + raw := `{"old":{"id":"old","volumes":[{"name":"raw","mode":"rw"},{"name":"mounted","mount":"/data"}],"volumes_want":[]},"desired":{"id":"desired","volumes":[],"volumes_want":[{"name":"raw"}]},"missing":{"id":"missing","volumes":[{"name":"raw"}]}}` + if err := os.WriteFile(filepath.Join(dir, "claims.json"), []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + s := newClaimStore(dir, false) + claims, err := s.load() + if err != nil { + t.Fatal(err) + } + if !claims["old"].Volumes[0].AttachOnly || claims["old"].Volumes[1].AttachOnly || claims["old"].VolumesWant == nil || len(claims["old"].VolumesWant) != 0 { + t.Fatalf("old shapes changed: %+v", claims["old"]) + } + if !claims["desired"].VolumesWant[0].AttachOnly || !slices.Equal(claims["missing"].Volumes, claims["missing"].VolumesWant) { + t.Fatal("legacy desired or missing-desired fallback changed") + } + refs := s.recoveryVolumes() + if !refs["old"][0].AttachOnly || !refs["desired"][0].AttachOnly { + t.Fatalf("recovery references captured before canonicalization: %+v", refs) + } + if err = os.WriteFile(filepath.Join(dir, "claims.json"), []byte(`{"bad":{"id":"bad","volumes":[{"name":"data","mount":"/data","attach_only":true}]}}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err = newClaimStore(dir, false).load(); err == nil { + t.Fatal("accepted contradictory persisted shape") + } +} diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 52348967..729cd6c9 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -152,7 +152,7 @@ type Engine interface { TrimCow(ctx context.Context, vsockSocket string) error CheckLiveVolumeSupport(ctx context.Context) error DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error - DetachVolume(ctx context.Context, vmName, sock, name, mount string) error + DetachVolume(ctx context.Context, vmName, sock, name, mount string) (engine.VolumeDetachResult, error) MountVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error MountLiveVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error UnmountVolume(ctx context.Context, vsockSocket, mount string) error diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 136cd118..585f2ed6 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1658,24 +1658,28 @@ func (f *fakeEngine) MountLiveVolume(ctx context.Context, sock, name, mount stri return f.MountVolume(ctx, sock, name, mount, rw) } -func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) error { +func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) (engine.VolumeDetachResult, error) { if f.liveVolumeHook != nil { if err := f.liveVolumeHook("detach"); err != nil { - return err + return engine.VolumeDetachResult{}, err } } if err := ctx.Err(); err != nil { - return err + return engine.VolumeDetachResult{}, err } if mount != "" { if err := f.UnmountVolume(ctx, sock, mount); err != nil { - return err + return engine.VolumeDetachResult{}, err } } f.mu.Lock() defer f.mu.Unlock() f.volumeOps = append(f.volumeOps, "detach:"+name) - return f.detachVolumeErr + if f.detachVolumeErr != nil { + return engine.VolumeDetachResult{}, f.detachVolumeErr + } + normal := slices.ContainsFunc(f.volumeMounts, func(v types.Volume) bool { return v.Name == name && v.Mount == mount && v.RW() }) + return engine.VolumeDetachResult{NormalUnmount: normal}, nil } func (f *fakeEngine) UnmountVolume(_ context.Context, _, mount string) error { diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 2362ab22..9d6bc177 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -140,8 +140,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if len(requested) == 0 { return nil, nil } - // an attach-only claim has no mounts to default; re-defaulting here would mount them - applied, err := types.ValidateVolumes(requested, types.VolumesAttachOnly(requested)) + applied, err := types.ValidateVolumes(requested, false) if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } @@ -188,7 +187,6 @@ func (m *Manager) resolveCatalogEntry(ctx context.Context, volume types.Volume, } disk := entry.disk disk.RW = volume.RW() - volume.AttachOnly = false // past validation the empty mount carries it return resolvedVolume{generation: entry.generation, disk: disk, applied: volume}, nil } diff --git a/sandboxd/pool/volume_attach_test.go b/sandboxd/pool/volume_attach_test.go index 7fc14e75..4926562f 100644 --- a/sandboxd/pool/volume_attach_test.go +++ b/sandboxd/pool/volume_attach_test.go @@ -21,7 +21,7 @@ func TestAttachOnlyClaimAttachesWithoutMounting(t *testing.T) { if err != nil { t.Fatalf("ClaimProvision: %v", err) } - want := []types.Volume{{Name: "scratch", Mode: types.VolumeModeRW}} + want := []types.Volume{{Name: "scratch", Mode: types.VolumeModeRW, AttachOnly: true}} if !slices.Equal(sb.Volumes, want) { t.Errorf("applied volumes=%v, want %v", sb.Volumes, want) } @@ -78,11 +78,7 @@ func TestAttachOnlyClaimAttachesEveryVolumeConcurrently(t *testing.T) { if err != nil { t.Fatalf("ClaimProvision: %v", err) } - want := []types.Volume{ - {Name: "scratch", Mode: types.VolumeModeRW}, - {Name: "dataset"}, - {Name: "cache", Mode: types.VolumeModeRW}, - } + want := slices.Clone(requested) if !slices.Equal(sb.Volumes, want) { t.Errorf("volumes=%v, want the request order %v", sb.Volumes, want) } diff --git a/sandboxd/pool/volume_modes_test.go b/sandboxd/pool/volume_modes_test.go new file mode 100644 index 00000000..282e4ed6 --- /dev/null +++ b/sandboxd/pool/volume_modes_test.go @@ -0,0 +1,387 @@ +package pool + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestLiveVolumeFourModesAndMixedRoundtrip(t *testing.T) { + eng := newFakeEngine() + catalog := []config.VolumeSpec{} + request := []types.Volume{{Name: "mounted"}, {Name: "writer", Mode: types.VolumeModeRW}, {Name: "raw", AttachOnly: true}, {Name: "raw-writer", Mode: types.VolumeModeRW, AttachOnly: true}} + for _, v := range request { + catalog = append(catalog, config.VolumeSpec{Name: v.Name, Path: writeVolumeImage(t, v.Name, "seed"), Writable: true}) + } + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + view, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: request}) + if err != nil { + t.Fatal(err) + } + if len(view.Volumes) != 4 || len(eng.volumeMounts) != 2 { + t.Fatalf("mixed modes: view=%+v mounts=%+v", view, eng.volumeMounts) + } + for _, v := range view.Volumes { + if v.AttachOnly != (v.Mount == "") { + t.Fatalf("lost canonical shape: %+v", v) + } + } + raw, err := json.Marshal(view) + if err != nil { + t.Fatal(err) + } + var echoed types.SandboxVolumesResponse + if err = json.Unmarshal(raw, &echoed); err != nil { + t.Fatal(err) + } + before := slices.Clone(eng.volumeOpsLog()) + if _, err = m.SetVolumes(t.Context(), sb.ID, "", echoed.VolumesWant); err != nil { + t.Fatal(err) + } + if !slices.Equal(before, eng.volumeOpsLog()) { + t.Fatal("GET-to-PUT roundtrip changed devices") + } + for _, change := range []types.Volume{{Name: "raw", Mount: "/new"}, {Name: "raw-writer", AttachOnly: true}, {Name: "writer", Mount: "/other", Mode: types.VolumeModeRW}} { + if _, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{change}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("held shape change accepted: %+v, %v", change, err) + } + } + legacy := slices.Clone(view.VolumesWant) + for i := range legacy { + legacy[i].AttachOnly = false + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", legacy); err != nil { + t.Fatalf("legacy held omission: %v", err) + } + assertLiveJournal(t, m, sb) + restarted := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err = restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + recovered, err := restarted.GetVolumes(sb.ID, "") + if err != nil || !slices.Equal(recovered.Volumes, view.Volumes) { + t.Fatalf("mixed restart: %+v, %v", recovered, err) + } + if _, err = restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "raw-writer", AttachOnly: true}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("restart lost writer exclusion: %v", err) + } + if _, err = restarted.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + for _, spec := range catalog { + if volumeDirty(spec.Path) || volumeHoldersOf(restarted, spec.Name) != (volumeHolders{}) { + t.Fatalf("detach leaked marker or hold: %s", spec.Name) + } + } +} + +func TestLiveWritableDetachRequiresCleanEvidence(t *testing.T) { + for _, proof := range []bool{false, true} { + t.Run(fmt.Sprint(proof), func(t *testing.T) { + base := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, base, catalog) + m.eng = &detachProofEngine{fakeEngine: base, proof: proof} + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err != nil { + t.Fatal(err) + } + path := catalog[0].Path + if !volumeDirty(path) || !base.attachDirty["data"] { + t.Fatal("writer was attached before dirty marker") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if volumeDirty(path) == proof { + t.Fatalf("marker disagrees with clean evidence: proof=%t dirty=%t", proof, volumeDirty(path)) + } + _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if proof && err != nil || !proof && !errors.Is(err, ErrVolumeNeedsRecovery) { + t.Fatalf("reader admission after detach: %v", err) + } + }) + } +} + +func TestLiveWritableFailedHostDetachKeepsMarkerAndHold(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err != nil { + t.Fatal(err) + } + eng.detachVolumeErr = errors.New("host removal uncertain") + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("failed host detach reported success") + } + if !volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data").writers != 1 { + t.Fatal("failed host detach lost dirty marker or writer") + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "uncertain", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture after uncertain detach: %v", err) + } + eng.detachVolumeErr = nil + m.eng = &detachProofEngine{fakeEngine: eng} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if !volumeDirty(catalog[0].Path) { + t.Fatal("absent-device retry invented clean evidence") + } +} + +func TestLiveWritableMarkerClearFailureRemainsConservative(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err != nil { + t.Fatal(err) + } + marker := volumeDirtyPath(catalog[0].Path) + if err := os.Remove(marker); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(marker, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(marker, "block"), nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if !volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("failed marker deletion changed physical retirement or reported clean") + } +} + +type detachProofEngine struct { + *fakeEngine + proof bool +} + +func (e *detachProofEngine) DetachVolume(ctx context.Context, vm, sock, name, mount string) (engine.VolumeDetachResult, error) { + _, err := e.fakeEngine.DetachVolume(ctx, vm, sock, name, mount) + if err != nil { + return engine.VolumeDetachResult{}, err + } + return engine.VolumeDetachResult{NormalUnmount: e.proof}, nil +} + +func TestLiveWritableUncertainAdditionClearsOnlyAfterDetach(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + eng.mountVolumeErr = errors.New("mount response lost") + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err == nil { + t.Fatal("expected unknown mount result") + } + if len(sb.Volumes) != 0 || len(sb.VolumesWant) != 1 || !volumeDirty(catalog[0].Path) { + t.Fatal("pending writable attachment lost intent or marker") + } + eng.mountVolumeErr = nil + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("uncertain addition cleanup retained marker or hold") + } +} + +func TestLiveVolumeModesKeepRecoveryAfterDetachWriteFailure(t *testing.T) { + for _, v := range []types.Volume{{Name: "data"}, {Name: "data", Mode: types.VolumeModeRW}, {Name: "data", AttachOnly: true}, {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} { + t.Run(fmt.Sprintf("rw=%t/raw=%t", v.RW(), v.AttachOnly), func(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{v}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 { + t.Fatalf("detach result failure: %+v, %v", view, err) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("writer passed unretired %v: %v", v, err) + } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture passed unretired %v: %v", v, err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + restarted := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err = restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = restarted.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if _, err = restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("retired reference still blocks writer: %v", err) + } + }) + } +} + +func TestLiveWritableReleaseWaitsForLateDetachBeforeNewWriter(t *testing.T) { + base := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, base, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err != nil { + t.Fatal(err) + } + eng := &gatedDetachEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} + m.eng = eng + changed := make(chan error, 1) + go func() { _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); changed <- err }() + <-eng.entered + resumed := false + defer func() { + if !resumed { + close(eng.resume) + <-changed + } + }() + if !volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data").writers != 1 { + t.Fatal("host result released marker or hold before consumption") + } + if !m.mu.TryLock() { + t.Fatal("host call holds manager mutex") + } + m.mu.Unlock() + if !sb.Transition.TryLock() { + t.Fatal("host call holds transition mutex") + } + sb.Transition.Unlock() + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("writer overtook canceled detach worker: %v", err) + } + close(eng.resume) + resumed = true + if err := <-changed; err == nil { + t.Fatal("released claim consumed late detach as successful mutation") + } + m.retryRemovals(t.Context()).Wait() + m.eng = base + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatal(err) + } + m.retryRemovals(t.Context()).Wait() + if !volumeDirty(catalog[0].Path) { + t.Fatal("old cleanup erased new writer marker") + } +} + +type gatedDetachEngine struct { + *fakeEngine + entered chan struct{} + resume chan struct{} +} + +func (e *gatedDetachEngine) DetachVolume(ctx context.Context, vm, sock, name, mount string) (engine.VolumeDetachResult, error) { + result, err := e.fakeEngine.DetachVolume(ctx, vm, sock, name, mount) + close(e.entered) + <-e.resume + return result, err +} + +func TestLiveVolumeModesRetryFailedIntentAndAttachWrites(t *testing.T) { + for _, stage := range []string{"intent", "attached"} { + for _, v := range []types.Volume{{Name: "data"}, {Name: "data", Mode: types.VolumeModeRW}, {Name: "data", AttachOnly: true}, {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} { + t.Run(fmt.Sprintf("%s/rw=%t/raw=%t", stage, v.RW(), v.AttachOnly), func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + m.store.recommitting.Store(true) + fail := func([]byte) error { return errors.New("claim write unavailable") } + if stage == "intent" { + m.store.replaceFile = fail + } else { + eng.liveVolumeHook = func(verb string) error { + if verb == "attach" { + m.store.replaceFile = fail + } + return nil + } + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{v}); err == nil { + t.Fatal("failed claim write reported success") + } + if len(sb.VolumesWant) != 1 || sb.VolumesWant[0].AttachOnly != v.AttachOnly { + t.Fatal("failed write lost per-entry intent") + } + if stage == "intent" && len(eng.volumeSpecs) != 0 { + t.Fatal("failed intent reached device call") + } + eng.liveVolumeHook = nil + m.store.replaceFile = nil + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{v}); err != nil { + t.Fatalf("same target failed to recover: %v", err) + } + assertLiveJournal(t, m, sb) + holds := volumeHoldersOf(m, "data") + if holds.readers+holds.writers != 1 { + t.Fatalf("retry duplicated hold: %+v", holds) + } + }) + } + } +} + +func TestLiveVolumeModesCanceledAttachRetainsRetryableIntent(t *testing.T) { + for _, v := range []types.Volume{{Name: "data"}, {Name: "data", Mode: types.VolumeModeRW}, {Name: "data", AttachOnly: true}, {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} { + t.Run(fmt.Sprintf("rw=%t/raw=%t", v.RW(), v.AttachOnly), func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + eng := newGatedVolumeEngine(base) + m.eng = eng + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { _, err := m.SetVolumes(ctx, sb.ID, "", []types.Volume{v}); done <- err }() + <-eng.entered + cancel() + close(eng.resume) + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("canceled attachment outcome: %v", err) + } + if len(sb.VolumesWant) != 1 || volumeHoldersOf(m, "data").readers+volumeHoldersOf(m, "data").writers != 1 { + t.Fatal("canceled attachment dropped intent or hold") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{v}); err != nil { + t.Fatalf("canceled request retry: %v", err) + } + assertLiveJournal(t, m, sb) + }) + } +} diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 407c4e0b..2e0e02db 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -7,6 +7,8 @@ import ( "maps" "slices" + "github.com/projecteru2/core/log" + "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -40,7 +42,7 @@ func (m *Manager) SetVolumes(ctx context.Context, id, tenant string, want []type return m.changeVolumes(ctx, id, tenant, func([]types.Volume) ([]types.Volume, error) { return slices.Clone(want), nil }) } -// PatchVolumes adds and removes a running claim's read-only mounted volumes by name. +// PatchVolumes adds and removes a running claim's volumes by name. func (m *Manager) PatchVolumes(ctx context.Context, id, tenant string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { return m.changeVolumes(ctx, id, tenant, func(current []types.Volume) ([]types.Volume, error) { if err := validateDetachNames(patch.Remove); err != nil { @@ -302,13 +304,29 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run } func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume) error { - if err := m.checkVolumeRun(ctx, sb, run); err != nil { + m.mu.Lock() + err := m.checkVolumeRunLocked(ctx, sb, run) + path := "" + if entry := m.catalog.entries[v.Name]; entry != nil { + path = entry.disk.Path + } + m.mu.Unlock() + if err != nil { return err } - if err := m.eng.DetachVolume(ctx, sb.VMName, sb.VsockSocket, v.Name, v.Mount); err != nil { + result, err := m.eng.DetachVolume(ctx, sb.VMName, sb.VsockSocket, v.Name, v.Mount) + if err != nil { return fmt.Errorf("detach volume %q: %w", v.Name, err) } - return m.checkVolumeRun(ctx, sb, run) + if err = m.checkVolumeRun(ctx, sb, run); err != nil { + return err + } + if needsUnmount(v) && result.NormalUnmount && path != "" { + if err = clearVolumeDirty(path); err != nil { + log.WithFunc("pool.detachVolumeRun").Errorf(ctx, err, "clear dirty marker for volume %s", v.Name) + } + } + return nil } func (m *Manager) commitVolumeApplied(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume, attached bool) error { @@ -422,29 +440,14 @@ func (m *Manager) clearVolumeError(sb *types.Sandbox, run *volumeRun, name strin } func normalizeLiveVolumes(volumes, held []types.Volume) ([]types.Volume, error) { - if len(volumes) > types.MaxClaimVolumes { - return nil, fmt.Errorf("%w: too many volumes", ErrBadVolume) - } - out := make([]types.Volume, 0, len(volumes)) - for _, v := range volumes { - attachOnly := v.AttachOnly - if i := volumeIndex(held, v.Name); i >= 0 && held[i].Mount == "" && v.Mount == "" { - attachOnly = true - } - normalized, err := types.ValidateVolumes([]types.Volume{v}, attachOnly) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) - } - if volumeIndex(out, v.Name) >= 0 { - return nil, fmt.Errorf("%w: duplicate volume %q", ErrBadVolume, v.Name) + request := slices.Clone(volumes) + for i, v := range request { + if old := volumeIndex(held, v.Name); old >= 0 && held[old].Mount == "" && v.Mount == "" { + request[i].AttachOnly = true } - if attachOnly { - normalized[0].AttachOnly = false - } - out = append(out, normalized[0]) } - mounted := slices.DeleteFunc(slices.Clone(out), func(v types.Volume) bool { return v.Mount == "" }) - if _, err := types.ValidateVolumes(mounted, false); err != nil { + out, err := types.ValidateVolumes(request, false) + if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } return out, nil @@ -456,18 +459,10 @@ func validateLiveTarget(held, target []types.Volume) error { if i >= 0 && target[i] != v { return fmt.Errorf("%w: remove volume %q before changing its mount or mode", ErrBadVolume, v.Name) } - if immutableVolume(v) && i < 0 { - return fmt.Errorf("%w: writable and attach-only volumes cannot change live", ErrBadVolume) - } - } - if slices.ContainsFunc(volumeDifference(target, held), immutableVolume) { - return fmt.Errorf("%w: live additions require read-only mounts", ErrBadVolume) } return nil } -func immutableVolume(v types.Volume) bool { return v.RW() || v.AttachOnly || v.Mount == "" } - func volumeIndex(volumes []types.Volume, name string) int { return slices.IndexFunc(volumes, func(v types.Volume) bool { return v.Name == name }) } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index c76afcc5..910bca56 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -68,8 +68,8 @@ func TestLiveAttachPreservesWritableAndAttachOnlyVolumes(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "other", Mount: original.Mount}}); !errors.Is(err, ErrBadVolume) { t.Fatalf("accepted mode change: %v", err) } - if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "other"}}); !errors.Is(err, ErrBadVolume) || len(sb.Volumes) != 2 { - t.Fatalf("writable detach changed the batch: %v", err) + if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "other"}}); err != nil || len(sb.Volumes) != 0 { + t.Fatalf("writable detach failed: %v", err) } }) } @@ -80,8 +80,8 @@ func TestLiveVolumeValidationPrecedesDeviceCalls(t *testing.T) { m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) for _, request := range [][]types.Volume{ - {{Name: "data", Mode: "rw"}}, - {{Name: "data", AttachOnly: true}}, + {{Name: "data", Mode: "unknown"}}, + {{Name: "data", AttachOnly: true, Mount: "/data"}}, {{Name: "data", Mount: "/etc"}}, {{Name: "data"}, {Name: "data"}}, {{Name: "data", Mount: "/a"}, {Name: "other", Mount: "/a/b"}}, diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 48ff46b6..98d4756d 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -171,3 +171,45 @@ func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { }) } } + +func TestLiveVolumeRoutesPreserveMixedAttachOnlyRoundtrip(t *testing.T) { + want := []types.Volume{{Name: "mounted", Mount: "/volumes/mounted"}, {Name: "raw", Mode: types.VolumeModeRW, AttachOnly: true}} + view := types.SandboxVolumesResponse{VolumesWant: want, Volumes: want} + mgr := &fakeManager{ + getVolumes: func(string, string) (types.SandboxVolumesResponse, error) { return view, nil }, + setVolumes: func(_, _ string, got []types.Volume) (types.SandboxVolumesResponse, error) { + if !slices.Equal(got, want) { + t.Errorf("GET-to-PUT changed mixed intent: %+v", got) + } + return view, nil + }, + patchVolumes: func(_, _ string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { + if !slices.Equal(patch.Add, want) { + t.Errorf("PATCH lost per-entry intent: %+v", patch.Add) + } + return view, nil + }, + } + ts := newTenantTestServer(t, "root", nil, mgr, nil) + get := doReq(t, http.MethodGet, ts.URL+"/v1/sandboxes/sb_1/volumes", "root", "") + defer get.Body.Close() + var got types.SandboxVolumesResponse + if err := json.UnmarshalRead(get.Body, &got); err != nil || !slices.Equal(got.VolumesWant, want) { + t.Fatalf("GET lost explicit attach-only flag: %+v, %v", got, err) + } + for _, method := range []string{http.MethodPut, http.MethodPatch} { + key := "volumes" + if method == http.MethodPatch { + key = "add" + } + body, err := json.Marshal(map[string][]types.Volume{key: got.VolumesWant}) + if err != nil { + t.Fatal(err) + } + resp := doReq(t, method, ts.URL+"/v1/sandboxes/sb_1/volumes", "root", string(body)) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("%s status=%d", method, resp.StatusCode) + } + } +} diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 4cc8d307..523c354c 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -752,8 +752,8 @@ type Volume struct { Name string `json:"name"` Mount string `json:"mount,omitempty"` Mode string `json:"mode,omitempty"` - // AttachOnly carries the claim's flag into validation; past it the empty Mount is the signal. - AttachOnly bool `json:"-"` + // AttachOnly leaves mounting and unmounting to the caller. + AttachOnly bool `json:"attach_only,omitzero"` } // RW reports whether the entry asks for write access. @@ -803,11 +803,6 @@ func VolumeRWNames(volumes []Volume) []string { return names } -// VolumesAttachOnly reports whether the caller mounts the devices itself. -func VolumesAttachOnly(volumes []Volume) bool { - return slices.ContainsFunc(volumes, func(v Volume) bool { return v.AttachOnly }) -} - // ValidateVolumes returns detached entries with defaults filled and modes normalized. func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { if len(volumes) > MaxClaimVolumes { @@ -834,7 +829,7 @@ func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { default: return nil, fmt.Errorf("volumes[%d] mode %q must be %s or %s", i, mode, VolumeModeRO, VolumeModeRW) } - if attachOnly { + if attachOnly || volume.AttachOnly { if volume.Mount != "" { return nil, fmt.Errorf("volumes[%d] mount %q is meaningless when the caller mounts the device itself", i, volume.Mount) } @@ -847,6 +842,9 @@ func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { } for j := range i { other := applied[j].Mount + if other == "" { + continue + } switch { case mount == other: return nil, fmt.Errorf("volumes[%d] mount %q duplicates volumes[%d]", i, mount, j) diff --git a/sandboxd/types/volume_test.go b/sandboxd/types/volume_test.go index 9549af49..f1c2dbbe 100644 --- a/sandboxd/types/volume_test.go +++ b/sandboxd/types/volume_test.go @@ -127,12 +127,6 @@ func TestValidateVolumesAttachOnly(t *testing.T) { if !slices.Equal(got, want) { t.Errorf("volumes %v, want %v", got, want) } - if !VolumesAttachOnly(got) { - t.Error("validated attach-only set does not report itself") - } - if VolumesAttachOnly(nil) || VolumesAttachOnly([]Volume{{Name: "dataset", Mount: "/volumes/dataset"}}) { - t.Error("mounted set reports itself attach-only") - } for _, tt := range []struct { name string @@ -173,3 +167,22 @@ func TestValidateVolumesRejectsGuestOSMounts(t *testing.T) { } } } + +func TestValidateMixedVolumesPreservesPerEntryIntent(t *testing.T) { + request := []Volume{{Name: "mounted"}, {Name: "writer", Mode: VolumeModeRW}, {Name: "raw", AttachOnly: true}, {Name: "raw-writer", Mode: VolumeModeRW, AttachOnly: true}} + got, err := ValidateVolumes(request, false) + if err != nil { + t.Fatal(err) + } + want := []Volume{{Name: "mounted", Mount: "/volumes/mounted"}, {Name: "writer", Mount: "/volumes/writer", Mode: VolumeModeRW}, {Name: "raw", AttachOnly: true}, {Name: "raw-writer", Mode: VolumeModeRW, AttachOnly: true}} + if !slices.Equal(got, want) { + t.Fatalf("mixed intent changed: got %+v, want %+v", got, want) + } + again, err := ValidateVolumes(got, false) + if err != nil || !slices.Equal(again, want) { + t.Fatalf("second validation changed mixed intent: %+v, %v", again, err) + } + if _, err = ValidateVolumes([]Volume{{Name: "raw", AttachOnly: true, Mount: "/data"}}, false); err == nil { + t.Fatal("accepted attach-only with a mount") + } +} From e8e9c23d0a839bb2c93bf4e9791e7e9afba23fb9 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 02:03:03 +0000 Subject: [PATCH 17/72] sdk: preserve per-volume attach-only intent --- sdk/go/client.go | 3 ++ sdk/go/options.go | 5 ++- sdk/go/volume_mutation_test.go | 44 +++++++++++++++++++- sdk/go/volumes.go | 4 +- sdk/go/volumes_test.go | 56 ++++++++++++++++++++++++++ sdk/python/cocoonsandbox/client.py | 3 +- sdk/python/cocoonsandbox/sandbox.py | 18 ++++----- sdk/python/cocoonsandbox/template.py | 8 ++-- sdk/python/cocoonsandbox/volumes.py | 16 +++++--- sdk/python/tests/test_client.py | 42 +++++++++++++++++++ sdk/python/tests/test_volumes.py | 60 +++++++++++++++++++++++++++- 11 files changed, 233 insertions(+), 26 deletions(-) diff --git a/sdk/go/client.go b/sdk/go/client.go index ccb932fd..582d19d4 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -115,6 +115,9 @@ func (r claimRequest) validateVolumes() error { if v.Mode != "" && v.Mode != volumeModeRW { return fmt.Errorf("volume %q: mode must be \"\", %q, or %q, got %q", v.Name, volumeModeRO, volumeModeRW, v.Mode) } + if v.AttachOnly && v.Mount != "" { + return fmt.Errorf("volume %q: mount %q is meaningless with attach_only, which leaves mounting to the caller", v.Name, v.Mount) + } if r.VolumesAttachOnly && v.Mount != "" { return fmt.Errorf("volume %q: mount %q is meaningless with WithVolumesAttachOnly, which leaves mounting to the caller", v.Name, v.Mount) } diff --git a/sdk/go/options.go b/sdk/go/options.go index 20b75401..277831ad 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -29,12 +29,13 @@ type NetShape string // node's warm pools. type Size string -// Volume requests one catalog entry at an optional guest mount path and mode. +// Volume requests a catalog entry. AttachOnly leaves mounting and unmounting to the caller. type Volume struct { Name string `json:"name"` Mount string `json:"mount,omitempty"` // Mode is "rw" for a writable mount; empty (or "ro") means read-only. - Mode string `json:"mode,omitempty"` + Mode string `json:"mode,omitempty"` + AttachOnly bool `json:"attach_only,omitzero"` } // VolumeInfo describes one visible fleet catalog entry. diff --git a/sdk/go/volume_mutation_test.go b/sdk/go/volume_mutation_test.go index 7f65427c..44fc937b 100644 --- a/sdk/go/volume_mutation_test.go +++ b/sdk/go/volume_mutation_test.go @@ -51,7 +51,7 @@ func TestLiveVolumesUseOwnerAndAPIToken(t *testing.T) { func TestLiveVolumeFailureRetainsPartialView(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) - _, _ = io.WriteString(w, `{"volumes_want":[{"name":"new","mount":"/new"}],"volumes":[{"name":"old","mount":"/old"}],"converging":false,"states":[{"name":"new","state":"pending","error":"retry"},{"name":"old","state":"detaching"}],"error":"convergence failed"}`) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"new","mode":"rw","attach_only":true}],"volumes":[{"name":"old","attach_only":true}],"converging":false,"states":[{"name":"new","state":"pending","error":"retry"},{"name":"old","state":"detaching"}],"error":"convergence failed"}`) })) t.Cleanup(ts.Close) c := testClient(t, ts) @@ -63,6 +63,11 @@ func TestLiveVolumeFailureRetainsPartialView(t *testing.T) { if !isVolume || !isAPI || api.Status != http.StatusInternalServerError || failure.Response == nil { t.Fatalf("%s error lost response or APIError: %v", method, err) } + for _, entries := range [][]Volume{got.Volumes, got.VolumesWant, sb.Volumes, sb.VolumesWant} { + if len(entries) != 1 || !entries[0].AttachOnly { + t.Fatalf("%s partial view lost attach_only: %+v", method, entries) + } + } if !reflect.DeepEqual(got, *failure.Response) || got.States[0].State != "pending" || got.States[1].State != "detaching" || sb.VolumesWant[0].Name != "new" || sb.Volumes[0].Name != "old" { t.Fatalf("partial view=%+v handle=%+v", got, sb) } @@ -188,3 +193,40 @@ func callVolumes(t *testing.T, sb *Sandbox, method string) (SandboxVolumesRespon return sb.GetVolumes(t.Context()) } } + +func TestLiveVolumeAttachOnlyRoundTrip(t *testing.T) { + const entries = `[{"name":"mounted-ro","mount":"/volumes/mounted-ro"},{"name":"mounted-rw","mount":"/work","mode":"rw"},{"name":"device-ro","attach_only":true},{"name":"device-rw","mode":"rw","attach_only":true}]` + var forwarded []map[string]any + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut { + var body struct { + Volumes []map[string]any `json:"volumes"` + } + if err := json.UnmarshalRead(r.Body, &body); err != nil { + t.Errorf("decode replacement: %v", err) + } + forwarded = body.Volumes + } + _, _ = io.WriteString(w, `{"volumes_want":`+entries+`,"volumes":`+entries+`,"states":[],"converging":false}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + sb := &Sandbox{ID: "sb_1", owner: c.addr, c: c} + view, err := sb.GetVolumes(t.Context()) + if err != nil { + t.Fatal(err) + } + if _, err = sb.SetVolumes(t.Context(), view.VolumesWant); err != nil { + t.Fatal(err) + } + var want []map[string]any + if err = json.Unmarshal([]byte(entries), &want); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(forwarded, want) { + t.Fatalf("GET-to-PUT changed volume intent: got=%v want=%v", forwarded, want) + } + if !slices.Equal(sb.Volumes, view.Volumes) || !slices.Equal(sb.VolumesWant, view.VolumesWant) { + t.Fatal("roundtrip changed handle state") + } +} diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go index 69e1dea4..0a4917fe 100644 --- a/sdk/go/volumes.go +++ b/sdk/go/volumes.go @@ -55,7 +55,7 @@ func (s *Sandbox) GetVolumes(ctx context.Context) (SandboxVolumesResponse, error return s.requestVolumes(ctx, http.MethodGet, nil, "get volumes") } -// SetVolumes converges the full desired set; an explicit empty slice clears mutable mounts, and nil is rejected. +// SetVolumes converges the full desired set; an explicit empty slice clears volumes, and nil is rejected. func (s *Sandbox) SetVolumes(ctx context.Context, volumes []Volume) (SandboxVolumesResponse, error) { if volumes == nil { return SandboxVolumesResponse{}, errors.New("set volumes: use an explicit empty slice to clear volumes") @@ -65,7 +65,7 @@ func (s *Sandbox) SetVolumes(ctx context.Context, volumes []Volume) (SandboxVolu }{Volumes: volumes}, "set volumes") } -// PatchVolumes adds or removes read-only mounted entries by name, retaining immutable entries. +// PatchVolumes adds or removes catalog volumes by name. func (s *Sandbox) PatchVolumes(ctx context.Context, patch VolumePatch) (SandboxVolumesResponse, error) { return s.requestVolumes(ctx, http.MethodPatch, patch, "patch volumes") } diff --git a/sdk/go/volumes_test.go b/sdk/go/volumes_test.go index d8a2c14e..c21ce4fa 100644 --- a/sdk/go/volumes_test.go +++ b/sdk/go/volumes_test.go @@ -3,6 +3,7 @@ package sandbox import ( "context" "encoding/json/v2" + "fmt" "net/http" "net/http/httptest" "slices" @@ -315,3 +316,58 @@ func TestSandboxVolumesEchoMode(t *testing.T) { t.Errorf("volumes = %+v, want %+v", sb.Volumes, want) } } + +func TestPerEntryAttachOnlyClaimModes(t *testing.T) { + for _, promoted := range []bool{false, true} { + t.Run(fmt.Sprint(promoted), func(t *testing.T) { + want := []Volume{ + {Name: "mounted-ro"}, + {Name: "mounted-rw", Mount: "/work", Mode: "rw"}, + {Name: "device-ro", AttachOnly: true}, + {Name: "device-rw", Mode: "rw", AttachOnly: true}, + } + var got claimRequest + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.UnmarshalRead(r.Body, &got); err != nil { + t.Errorf("decode mixed claim: %v", err) + } + _ = json.MarshalWrite(w, claimResponse{ID: "sb_1", Token: "tok", Volumes: want}) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + var sb *Sandbox + var err error + if promoted { + tpl := &Template{Name: "task:v1", c: c, addr: c.addr, net: "none", size: "small"} + sb, err = tpl.New(t.Context(), WithVolumes(want...)) + } else { + sb, err = c.New(t.Context(), "rt:24.04", WithVolumes(want...)) + } + if err != nil { + t.Fatal(err) + } + if got.VolumesAttachOnly || !slices.Equal(got.Volumes, want) || !slices.Equal(sb.Volumes, want) { + t.Fatalf("mixed claim=%+v reply=%+v, want=%+v", got, sb.Volumes, want) + } + }) + } +} + +func TestPerEntryAttachOnlyRejectsClaimMount(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("invalid claim reached server") })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + for _, promoted := range []bool{false, true} { + var err error + option := WithVolumes(Volume{Name: "data", Mount: "/data", AttachOnly: true}) + if promoted { + tpl := &Template{Name: "task:v1", c: c, addr: c.addr} + _, err = tpl.New(t.Context(), option) + } else { + _, err = c.New(t.Context(), "rt:24.04", option) + } + if err == nil || !strings.Contains(err.Error(), "meaningless with attach_only") { + t.Fatalf("conflicting mount: %v", err) + } + } +} diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index aa3c8fd1..74ce4174 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -18,6 +18,7 @@ from .errors import APIError, SandboxTimeout, require_field from .sandbox import Sandbox from .template import _claim_body, _template_query +from .volumes import Volume T = TypeVar("T") @@ -60,7 +61,7 @@ def new( size: str = "", ttl_seconds: int = 0, claim_ref: str = "", - volumes: list[str | Mapping[str, str]] | None = None, + volumes: Sequence[str | Mapping[str, str | bool] | Volume] | None = None, mount: bool = True, *, deadline: float | None = None, diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 1a83fa86..12f695d5 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -7,7 +7,7 @@ import threading import time from collections import deque -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, cast from .checkpoint import Checkpoint @@ -45,7 +45,7 @@ def __init__( deadline: str = "", from_checkpoint: str = "", template_digest: str = "", - volumes: list[dict[str, str]] | None = None, + volumes: Sequence[Mapping[str, str | bool] | Volume] | None = None, ) -> None: self._client = client self.id = id @@ -54,7 +54,7 @@ def __init__( self.deadline = deadline self.from_checkpoint = from_checkpoint self.template_digest = template_digest - self.volumes = [dict(volume) for volume in volumes or []] + self.volumes = [dict(cast(Mapping[str, str | bool], volume)) for volume in volumes or []] self.volumes_want = [dict(volume) for volume in self.volumes] self._pool = ConnPool(client.keep_alive) self._proto = 0 @@ -74,9 +74,9 @@ def get_volumes(self, *, timeout: float | None = None) -> SandboxVolumesResponse return self._volumes_request("GET", None, "get volumes", timeout) def set_volumes( - self, volumes: list[str | Mapping[str, str] | Volume], *, timeout: float | None = None + self, volumes: Sequence[str | Mapping[str, str | bool] | Volume], *, timeout: float | None = None ) -> SandboxVolumesResponse: - """Replaces the desired set and converges; an explicit empty list clears mutable mounts.""" + """Replaces the desired set and converges; an explicit empty list clears volumes.""" if volumes is None or isinstance(volumes, str): raise TypeError("set volumes requires an explicit list; use [] to clear volumes") return self._volumes_request( @@ -86,11 +86,11 @@ def set_volumes( def patch_volumes( self, *, - add: list[str | Mapping[str, str] | Volume] | None = None, + add: Sequence[str | Mapping[str, str | bool] | Volume] | None = None, remove: list[str] | None = None, timeout: float | None = None, ) -> SandboxVolumesResponse: - """Adds or removes mounted read-only entries by name; omitted arguments leave the set unchanged.""" + """Adds or removes catalog volumes by name; omitted arguments leave the set unchanged.""" if isinstance(add, str) or isinstance(remove, str): raise TypeError("patch volumes takes lists of volumes and names") body: dict[str, Any] = {} @@ -445,8 +445,8 @@ def _volumes_request( return view def _record_volumes(self, view: SandboxVolumesResponse) -> None: - self.volumes_want = [cast(dict[str, str], volume.copy()) for volume in view["volumes_want"]] - self.volumes = [cast(dict[str, str], volume.copy()) for volume in view["volumes"]] + self.volumes_want = [cast(dict[str, str | bool], volume.copy()) for volume in view["volumes_want"]] + self.volumes = [cast(dict[str, str | bool], volume.copy()) for volume in view["volumes"]] def _dial(self, deadline: float | None = None) -> Conn: return self._client._dial(self.owner, self.id, self.token, deadline) diff --git a/sdk/python/cocoonsandbox/template.py b/sdk/python/cocoonsandbox/template.py index 5e468aa0..940ca290 100644 --- a/sdk/python/cocoonsandbox/template.py +++ b/sdk/python/cocoonsandbox/template.py @@ -3,10 +3,10 @@ from __future__ import annotations import urllib.parse -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any -from .volumes import _volume_body +from .volumes import Volume, _volume_body if TYPE_CHECKING: from .client import Client @@ -27,7 +27,7 @@ def __init__(self, client: Client, addr: str, name: str, net: str, size: str, co def new( self, ttl_seconds: int = 0, - volumes: list[str | Mapping[str, str]] | None = None, + volumes: Sequence[str | Mapping[str, str | bool] | Volume] | None = None, mount: bool = True, *, deadline: float | None = None, @@ -55,7 +55,7 @@ def _claim_body( net: str, size: str, ttl_seconds: int, - volumes: list[str | Mapping[str, str]] | None = None, + volumes: Sequence[str | Mapping[str, str | bool] | Volume] | None = None, mount: bool = True, claim_ref: str = "", ) -> dict[str, Any]: diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index ce156554..855cf21c 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -16,10 +16,11 @@ class _VolumeName(TypedDict): class Volume(_VolumeName, total=False): - """A catalog volume by name, with its guest mount point and an `rw` mode for writable claims.""" + """A catalog volume by name, with an optional mount, write mode, or caller-managed mounting.""" mount: str mode: str + attach_only: bool class _VolumeState(TypedDict): @@ -56,19 +57,24 @@ def _volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | N return cast(SandboxVolumesResponse, reply) -def _volume_body(volume: str | Mapping[str, str] | Volume, mount: bool) -> dict[str, str]: +def _volume_body(volume: str | Mapping[str, str | bool] | Volume, mount: bool) -> dict[str, str | bool]: if isinstance(volume, str): return {"name": volume} if not isinstance(volume, Mapping): raise TypeError("volume must be a name string or mapping") - unknown = sorted(set(volume) - {"name", "mount", "mode"}) + unknown = sorted(set(volume) - {"name", "mount", "mode", "attach_only"}) if unknown: raise TypeError( - f"volume mapping accepts only name, mount, and mode, got unexpected key(s): {', '.join(unknown)}" + "volume mapping accepts only name, mount, mode, and attach_only, " + f"got unexpected key(s): {', '.join(unknown)}" ) + if "attach_only" in volume and not isinstance(volume["attach_only"], bool): + raise TypeError("volume attach_only must be a boolean") + if volume.get("attach_only") and volume.get("mount"): + raise TypeError("volume mount is meaningless with attach_only=True, which leaves mounting to the caller") if not mount and "mount" in volume: raise TypeError("volume mount is meaningless with mount=False, which leaves mounting to the caller") - body = dict(cast(Mapping[str, str], volume)) + body = dict(cast(Mapping[str, str | bool], volume)) mode = body.get("mode") if mode in (None, "", "ro"): body.pop("mode", None) diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 90b799dc..7cf49848 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -382,3 +382,45 @@ def kept_node(): threading.Thread(target=server.serve_forever, daemon=True).start() yield f"127.0.0.1:{server.server_port}" server.shutdown() + + +@pytest.mark.parametrize("promoted", [False, True]) +def test_claim_preserves_mixed_volume_modes(node, promoted): + entries = [ + {"name": "mounted-ro"}, + {"name": "mounted-rw", "mount": "/work", "mode": "rw"}, + {"name": "device-ro", "attach_only": True}, + {"name": "device-rw", "mode": "rw", "attach_only": True}, + ] + seen = recording_claim({"id": "sb_1", "token": "tok", "volumes": entries}) + client = Client(node) + if promoted: + sb = Template(client, node, "task:v1", "none", "small").new(volumes=entries) + else: + sb = client.new("rt:24.04", volumes=entries) + assert seen[0]["volumes"] == entries + assert "volumes_attach_only" not in seen[0] + assert sb.volumes == entries + + +@pytest.mark.parametrize("promoted", [False, True]) +def test_claim_global_attach_only_keeps_per_entry_flags(node, promoted): + entries = [{"name": "data", "attach_only": False}, {"name": "work", "mode": "rw", "attach_only": True}] + seen = recording_claim({"id": "sb_1", "token": "tok"}) + client = Client(node) + if promoted: + Template(client, node, "task:v1", "none", "small").new(volumes=entries, mount=False) + else: + client.new("rt:24.04", volumes=entries, mount=False) + assert seen[0]["volumes_attach_only"] is True + assert seen[0]["volumes"] == entries + + +@pytest.mark.parametrize("promoted", [False, True]) +def test_claim_rejects_per_entry_attach_only_mount(node, promoted): + entries = [{"name": "data", "mount": "/data", "attach_only": True}] + with pytest.raises(TypeError, match="meaningless with attach_only=True"): + if promoted: + Template(Client(node), node, "task:v1", "none", "small").new(volumes=entries) + else: + Client(node).new("rt:24.04", volumes=entries) diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 3226b82d..3c62fc4a 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -46,8 +46,8 @@ def reply(body, path): @pytest.mark.parametrize("method", ["GET", "PUT", "PATCH"]) def test_volume_error_preserves_typed_partial_view(spawn_node, method): partial = { - "volumes_want": [{"name": "new", "mount": "/new"}], - "volumes": [{"name": "old", "mount": "/old"}], + "volumes_want": [{"name": "new", "mode": "rw", "attach_only": True}], + "volumes": [{"name": "old", "attach_only": True}], "converging": False, "states": [ {"name": "new", "state": "pending", "error": "retry"}, @@ -164,3 +164,59 @@ def test_incomplete_volume_view_preserves_handle(spawn_node, status): def test_volume_methods_reject_a_bare_string(black_hole, call): with pytest.raises(TypeError): call(sandbox_at(black_hole)) + + +def test_mixed_volume_modes_roundtrip(spawn_node): + entries = [ + {"name": "mounted-ro", "mount": "/volumes/mounted-ro"}, + {"name": "mounted-rw", "mount": "/work", "mode": "rw"}, + {"name": "device-ro", "attach_only": True}, + {"name": "device-rw", "mode": "rw", "attach_only": True}, + ] + view = {"volumes_want": entries, "volumes": entries, "states": [], "converging": False} + seen = [] + + def reply(body, path): + seen.append(body) + return 200, view + + sb = sandbox_at(spawn_node({("GET", PATH): reply, ("PUT", PATH): reply, ("PATCH", PATH): reply})) + got = sb.get_volumes() + sb.set_volumes(got["volumes_want"]) + sb.patch_volumes(add=got["volumes"]) + assert seen == [{}, {"volumes": entries}, {"add": entries}] + assert sb.volumes == sb.volumes_want == entries + + +@pytest.mark.parametrize("value", [None, 0, 1, "true", [], {}]) +@pytest.mark.parametrize("method", ["set", "patch"]) +def test_attach_only_requires_a_boolean(black_hole, value, method): + sb = sandbox_at(black_hole) + with pytest.raises(TypeError, match="attach_only must be a boolean"): + if method == "set": + sb.set_volumes([{"name": "data", "attach_only": value}]) + else: + sb.patch_volumes(add=[{"name": "data", "attach_only": value}]) + + +@pytest.mark.parametrize("method", ["set", "patch"]) +def test_attach_only_rejects_a_nonempty_mount(black_hole, method): + sb = sandbox_at(black_hole) + with pytest.raises(TypeError, match="meaningless with attach_only=True"): + if method == "set": + sb.set_volumes([{"name": "data", "mount": "/data", "attach_only": True}]) + else: + sb.patch_volumes(add=[{"name": "data", "mount": "/data", "attach_only": True}]) + + +def test_attach_only_empty_mount_and_false_flag_remain_explicit(spawn_node): + seen = [] + + def reply(body, path): + seen.append(body) + return 200, {"volumes_want": [], "volumes": [], "states": [], "converging": False} + + sb = sandbox_at(spawn_node({("PUT", PATH): reply})) + entries = [{"name": "device", "mount": "", "attach_only": True}, {"name": "mounted", "attach_only": False}] + sb.set_volumes(entries) + assert seen == [{"volumes": entries}] From 1ebc608765153c6b26d7b47818c816c9187a36e5 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 02:09:54 +0000 Subject: [PATCH 18/72] e2e: verify mixed live volume modes through the SDK --- e2e/e2e_test.go | 4 ++-- e2e/volume_mutation_test.go | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index cf305baa..a97279b6 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -764,7 +764,7 @@ func TestAttachOnlyVolumeEndToEnd(t *testing.T) { if err != nil { t.Fatalf("attach-only claim: %v", err) } - if want := []sandbox.Volume{{Name: "scratch", Mode: "rw"}}; !slices.Equal(sb.Volumes, want) { + if want := []sandbox.Volume{{Name: "scratch", Mode: "rw", AttachOnly: true}}; !slices.Equal(sb.Volumes, want) { t.Errorf("claim volumes %+v, want %+v", sb.Volumes, want) } applied := []string{"attach:scratch:rw"} @@ -800,7 +800,7 @@ func TestAttachOnlyVolumeWireShape(t *testing.T) { { "attach-only omits the mount", `{"template":"rt:24.04","volumes_attach_only":true,"volumes":[{"name":"scratch","mode":"rw"}]}`, - map[string]any{"name": "scratch", "mode": "rw"}, + map[string]any{"name": "scratch", "mode": "rw", "attach_only": true}, }, { "eager claim is unchanged", diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index ccd6d01d..91e69404 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -77,3 +77,45 @@ func TestEmptyVolumePatchChangesNothing(t *testing.T) { t.Fatalf("empty patch: %+v err=%v", got, err) } } + +func TestMixedLiveVolumeModesThroughSDK(t *testing.T) { + volumes := []sandbox.Volume{ + {Name: "mounted-ro"}, + {Name: "mounted-rw", Mode: "rw"}, + {Name: "device-ro", AttachOnly: true}, + {Name: "device-rw", Mode: "rw", AttachOnly: true}, + } + var catalog []config.VolumeSpec + for _, volume := range volumes { + catalog = append(catalog, config.VolumeSpec{ + Name: volume.Name, Path: writeVolumeImage(t, volume.Name+".img", "seed"), Writable: volume.Mode == "rw", + }) + } + stack := startTenantStack(t, "node-token", nil, catalog) + sb, err := stack.client.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer sb.Close() + view, err := sb.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: volumes}) + if err != nil { + t.Fatal(err) + } + want := slices.Clone(volumes) + want[0].Mount, want[1].Mount = "/volumes/mounted-ro", "/volumes/mounted-rw" + if !slices.Equal(view.Volumes, want) || !slices.Equal(sb.Volumes, want) { + t.Fatalf("mixed SDK attachment = %+v, handle=%+v, want %+v", view, sb.Volumes, want) + } + view, err = sb.GetVolumes(t.Context()) + if err != nil { + t.Fatal(err) + } + view, err = sb.SetVolumes(t.Context(), view.VolumesWant) + if err != nil || !slices.Equal(view.Volumes, want) || view.Converging { + t.Fatalf("mixed GET-to-PUT = %+v, err=%v", view, err) + } + view, err = sb.SetVolumes(t.Context(), []sandbox.Volume{}) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || view.Converging { + t.Fatalf("mixed removal = %+v, err=%v", view, err) + } +} From 55dc2230da6d02d9c1256697c441e256403d3ef8 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 02:20:53 +0000 Subject: [PATCH 19/72] sdk: validate Python volume inputs and retain Python 3.9 support --- sdk/python/cocoonsandbox/sandbox.py | 18 +++++------ sdk/python/cocoonsandbox/template.py | 8 +++-- sdk/python/cocoonsandbox/volumes.py | 15 +++++++-- sdk/python/tests/test_client.py | 46 ++++++++++++++++++++++++++++ sdk/python/tests/test_volumes.py | 38 +++++++++++++++++++++++ 5 files changed, 110 insertions(+), 15 deletions(-) diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 12f695d5..46666d34 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -24,7 +24,7 @@ ) from .frames import BULK_CHUNK, FS_CHUNK, KEEP_ALIVE_PROTO from .template import Template -from .volumes import SandboxVolumesResponse, Volume, _volume_body, _volume_response +from .volumes import SandboxVolumesResponse, Volume, _volume_bodies, _volume_response if TYPE_CHECKING: from .client import Client @@ -54,7 +54,7 @@ def __init__( self.deadline = deadline self.from_checkpoint = from_checkpoint self.template_digest = template_digest - self.volumes = [dict(cast(Mapping[str, str | bool], volume)) for volume in volumes or []] + self.volumes = [dict(cast("Mapping[str, str | bool]", volume)) for volume in volumes or []] self.volumes_want = [dict(volume) for volume in self.volumes] self._pool = ConnPool(client.keep_alive) self._proto = 0 @@ -77,11 +77,9 @@ def set_volumes( self, volumes: Sequence[str | Mapping[str, str | bool] | Volume], *, timeout: float | None = None ) -> SandboxVolumesResponse: """Replaces the desired set and converges; an explicit empty list clears volumes.""" - if volumes is None or isinstance(volumes, str): + if volumes is None: raise TypeError("set volumes requires an explicit list; use [] to clear volumes") - return self._volumes_request( - "PUT", {"volumes": [_volume_body(volume, True) for volume in volumes]}, "set volumes", timeout - ) + return self._volumes_request("PUT", {"volumes": _volume_bodies(volumes, True)}, "set volumes", timeout) def patch_volumes( self, @@ -91,11 +89,11 @@ def patch_volumes( timeout: float | None = None, ) -> SandboxVolumesResponse: """Adds or removes catalog volumes by name; omitted arguments leave the set unchanged.""" - if isinstance(add, str) or isinstance(remove, str): + if isinstance(remove, (str, bytes, bytearray)): raise TypeError("patch volumes takes lists of volumes and names") body: dict[str, Any] = {} if add is not None: - body["add"] = [_volume_body(volume, True) for volume in add] + body["add"] = _volume_bodies(add, True) if remove is not None: body["remove"] = list(remove) return self._volumes_request("PATCH", body, "patch volumes", timeout) @@ -445,8 +443,8 @@ def _volumes_request( return view def _record_volumes(self, view: SandboxVolumesResponse) -> None: - self.volumes_want = [cast(dict[str, str | bool], volume.copy()) for volume in view["volumes_want"]] - self.volumes = [cast(dict[str, str | bool], volume.copy()) for volume in view["volumes"]] + self.volumes_want = [cast("dict[str, str | bool]", volume.copy()) for volume in view["volumes_want"]] + self.volumes = [cast("dict[str, str | bool]", volume.copy()) for volume in view["volumes"]] def _dial(self, deadline: float | None = None) -> Conn: return self._client._dial(self.owner, self.id, self.token, deadline) diff --git a/sdk/python/cocoonsandbox/template.py b/sdk/python/cocoonsandbox/template.py index 940ca290..56a0ea05 100644 --- a/sdk/python/cocoonsandbox/template.py +++ b/sdk/python/cocoonsandbox/template.py @@ -6,7 +6,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any -from .volumes import Volume, _volume_body +from .volumes import Volume, _volume_bodies if TYPE_CHECKING: from .client import Client @@ -66,8 +66,10 @@ def _claim_body( claim["size"] = size if ttl_seconds: claim["ttl_seconds"] = ttl_seconds - if volumes: - claim["volumes"] = [_volume_body(volume, mount) for volume in volumes] + if volumes is not None: + entries = _volume_bodies(volumes, mount) + if entries: + claim["volumes"] = entries if not mount: claim["volumes_attach_only"] = True if claim_ref: diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index 855cf21c..a892d173 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Any, Final, Literal, TypedDict, cast VolumePhase = Literal["attached", "pending", "detaching"] @@ -57,6 +57,14 @@ def _volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | N return cast(SandboxVolumesResponse, reply) +def _volume_bodies( + volumes: Sequence[str | Mapping[str, str | bool] | Volume], mount: bool +) -> list[dict[str, str | bool]]: + if isinstance(volumes, (str, bytes, bytearray)) or not isinstance(volumes, Sequence): + raise TypeError("volumes must be a sequence of volume names or mappings") + return [_volume_body(volume, mount) for volume in volumes] + + def _volume_body(volume: str | Mapping[str, str | bool] | Volume, mount: bool) -> dict[str, str | bool]: if isinstance(volume, str): return {"name": volume} @@ -68,13 +76,16 @@ def _volume_body(volume: str | Mapping[str, str | bool] | Volume, mount: bool) - "volume mapping accepts only name, mount, mode, and attach_only, " f"got unexpected key(s): {', '.join(unknown)}" ) + for field in ("name", "mount", "mode"): + if field in volume and not isinstance(volume[field], str): + raise TypeError(f"volume {field} must be a string") if "attach_only" in volume and not isinstance(volume["attach_only"], bool): raise TypeError("volume attach_only must be a boolean") if volume.get("attach_only") and volume.get("mount"): raise TypeError("volume mount is meaningless with attach_only=True, which leaves mounting to the caller") if not mount and "mount" in volume: raise TypeError("volume mount is meaningless with mount=False, which leaves mounting to the caller") - body = dict(cast(Mapping[str, str | bool], volume)) + body = dict(cast("Mapping[str, str | bool]", volume)) mode = body.get("mode") if mode in (None, "", "ro"): body.pop("mode", None) diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 7cf49848..88b220cf 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -424,3 +424,49 @@ def test_claim_rejects_per_entry_attach_only_mount(node, promoted): Template(Client(node), node, "task:v1", "none", "small").new(volumes=entries) else: Client(node).new("rt:24.04", volumes=entries) + + +@pytest.mark.parametrize("promoted", [False, True]) +def test_claim_methods_reject_non_volume_sequences_before_request(node, promoted): + seen = recording_claim({"id": "sb_1", "token": "tok"}) + client = Client(node) + claim = ( + Template(client, node, "task:v1", "none", "small").new + if promoted + else lambda **kw: client.new("rt:24.04", **kw) + ) + for volumes in ("data", "", b"data", b"", bytearray(), {"name": "data"}): + with pytest.raises(TypeError): + claim(volumes=volumes) + assert seen == [] + + +@pytest.mark.parametrize("promoted", [False, True]) +@pytest.mark.parametrize("field", ["name", "mount", "mode"]) +def test_claim_methods_reject_non_string_volume_fields_before_request(node, promoted, field): + seen = recording_claim({"id": "sb_1", "token": "tok"}) + client = Client(node) + claim = ( + Template(client, node, "task:v1", "none", "small").new + if promoted + else lambda **kw: client.new("rt:24.04", **kw) + ) + volume = {"name": "data", field: True} + with pytest.raises(TypeError): + claim(volumes=[volume]) + assert seen == [] + + +@pytest.mark.parametrize("promoted", [False, True]) +def test_claim_methods_preserve_empty_volume_sequence_semantics(node, promoted): + seen = recording_claim({"id": "sb_1", "token": "tok"}) + client = Client(node) + claim = ( + Template(client, node, "task:v1", "none", "small").new + if promoted + else lambda **kw: client.new("rt:24.04", **kw) + ) + for volumes in (None, [], ()): + claim(volumes=volumes) + assert len(seen) == 3 + assert all("volumes" not in body for body in seen) diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 3c62fc4a..c35724a7 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -159,6 +159,7 @@ def test_incomplete_volume_view_preserves_handle(spawn_node, status): lambda sb: sb.set_volumes("data"), lambda sb: sb.patch_volumes(add="data"), lambda sb: sb.patch_volumes(remove="old"), + lambda sb: sb.patch_volumes(remove=b""), ], ) def test_volume_methods_reject_a_bare_string(black_hole, call): @@ -220,3 +221,40 @@ def reply(body, path): entries = [{"name": "device", "mount": "", "attach_only": True}, {"name": "mounted", "attach_only": False}] sb.set_volumes(entries) assert seen == [{"volumes": entries}] + + +@pytest.mark.parametrize("method", ["set", "patch"]) +@pytest.mark.parametrize("field", ["name", "mount", "mode"]) +def test_live_volume_methods_reject_non_string_fields_before_request(spawn_node, method, field): + seen = [] + + def reply(body, path): + seen.append(body) + return 200, {"volumes_want": [], "volumes": [], "states": [], "converging": False} + + sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) + volume = {"name": "data", field: True} + with pytest.raises(TypeError): + if method == "set": + sb.set_volumes([volume]) + else: + sb.patch_volumes(add=[volume]) + assert seen == [] + + +@pytest.mark.parametrize("method", ["set", "patch"]) +def test_live_volume_methods_reject_binary_collections_before_request(spawn_node, method): + seen = [] + + def reply(body, path): + seen.append(body) + return 200, {"volumes_want": [], "volumes": [], "states": [], "converging": False} + + sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) + for volumes in (b"data", b"", bytearray(), {"name": "data"}): + with pytest.raises(TypeError): + if method == "set": + sb.set_volumes(volumes) + else: + sb.patch_volumes(add=volumes) + assert seen == [] From 53fa13004878957120ff4a76c5b13e0c48946fb3 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 03:24:51 +0000 Subject: [PATCH 20/72] volumes: preserve live response compatibility and tighten detach proof --- e2e/e2e_test.go | 2 +- e2e/fakeengine_test.go | 40 ++++++-- e2e/volume_mutation_test.go | 60 +++++++++++ sandboxd/engine/volume_live.go | 25 +++-- sandboxd/engine/volume_live_test.go | 22 +++- sandboxd/pool/volume.go | 6 +- sandboxd/pool/volume_attach_test.go | 6 +- sandboxd/pool/volume_mutation.go | 48 +++++---- sandboxd/server/api.go | 15 +++ sandboxd/server/server.go | 2 +- sandboxd/server/server_http.go | 12 ++- sandboxd/server/volume_mutation_test.go | 14 ++- sandboxd/server/volume_wire_test.go | 127 ++++++++++++++++++++++++ sandboxd/types/types.go | 7 +- sandboxd/types/volume_test.go | 34 ++++--- 15 files changed, 348 insertions(+), 72 deletions(-) create mode 100644 sandboxd/server/volume_wire_test.go diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index a97279b6..44cf0f9a 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -800,7 +800,7 @@ func TestAttachOnlyVolumeWireShape(t *testing.T) { { "attach-only omits the mount", `{"template":"rt:24.04","volumes_attach_only":true,"volumes":[{"name":"scratch","mode":"rw"}]}`, - map[string]any{"name": "scratch", "mode": "rw", "attach_only": true}, + map[string]any{"name": "scratch", "mode": "rw"}, }, { "eager claim is unchanged", diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index d970831c..7b557be2 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -27,6 +27,7 @@ type fakeEngine struct { volumeOps []string volumeVMs map[string]bool guestEnvs map[string]string + mounts map[string]map[string]fakeVolumeMount seq int readerRoot string @@ -34,6 +35,11 @@ type fakeEngine struct { readerAttaches []string } +type fakeVolumeMount struct { + name string + rw bool +} + func newFakeEngine(dir string) *fakeEngine { return &fakeEngine{ real: engine.New("cocoon", nil, nil, false, false, ""), @@ -42,6 +48,7 @@ func newFakeEngine(dir string) *fakeEngine { socks: map[string]string{}, volumeVMs: map[string]bool{}, guestEnvs: map[string]string{}, + mounts: map[string]map[string]fakeVolumeMount{}, } } @@ -66,6 +73,7 @@ func (f *fakeEngine) Remove(_ context.Context, name string) error { _ = l.Close() } delete(f.listeners, name) + delete(f.mounts, f.socks[name]) delete(f.socks, name) if f.volumeVMs[name] { delete(f.volumeVMs, name) @@ -190,10 +198,14 @@ func (f *fakeEngine) DiskAttach(_ context.Context, vmName string, spec engine.Vo return nil } -func (f *fakeEngine) MountVolume(_ context.Context, _, name, mount string, rw bool) error { +func (f *fakeEngine) MountVolume(_ context.Context, sock, name, mount string, rw bool) error { f.mu.Lock() defer f.mu.Unlock() f.volumeOps = append(f.volumeOps, "mount:"+name+":"+mount+":"+volumeMode(rw)) + if f.mounts[sock] == nil { + f.mounts[sock] = map[string]fakeVolumeMount{} + } + f.mounts[sock][mount] = fakeVolumeMount{name: name, rw: rw} return nil } @@ -201,13 +213,18 @@ func (f *fakeEngine) MountLiveVolume(ctx context.Context, sock, name, mount stri return f.MountVolume(ctx, sock, name, mount, rw) } -func (f *fakeEngine) UnmountVolume(_ context.Context, _, mount string) error { +func (f *fakeEngine) UnmountVolume(_ context.Context, sock, mount string) error { f.mu.Lock() defer f.mu.Unlock() - f.volumeOps = append(f.volumeOps, "umount:"+mount) + f.unmountVolumeLocked(sock, mount) return nil } +func (f *fakeEngine) unmountVolumeLocked(sock, mount string) { + f.volumeOps = append(f.volumeOps, "umount:"+mount) + delete(f.mounts[sock], mount) +} + func (f *fakeEngine) SyncGuest(_ context.Context, _ string) error { f.mu.Lock() defer f.mu.Unlock() @@ -215,14 +232,21 @@ func (f *fakeEngine) SyncGuest(_ context.Context, _ string) error { return nil } -func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) (engine.VolumeDetachResult, error) { - if err := f.UnmountVolume(ctx, sock, mount); err != nil { - return engine.VolumeDetachResult{}, err - } +func (f *fakeEngine) DetachVolume(_ context.Context, _, sock, name, mount string) (engine.VolumeDetachResult, error) { f.mu.Lock() defer f.mu.Unlock() + entry, mounted := f.mounts[sock][mount] + normal := mount != "" && mounted && entry.name == name && entry.rw + if mount != "" && mounted && entry.name == name { + f.unmountVolumeLocked(sock, mount) + } + for _, other := range f.mounts[sock] { + if other.name == name { + return engine.VolumeDetachResult{}, &engine.VolumeMountConflict{Message: "volume still has a simulated guest mount"} + } + } f.volumeOps = append(f.volumeOps, "detach:"+name) - return engine.VolumeDetachResult{}, nil + return engine.VolumeDetachResult{NormalUnmount: normal}, nil } func (f *fakeEngine) volumeOpsLog() []string { diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index 91e69404..759f96b3 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -3,6 +3,7 @@ package e2e import ( "errors" "net/http" + "os" "slices" "strings" "testing" @@ -11,6 +12,65 @@ import ( sandbox "github.com/cocoonstack/sandbox/sdk/go" ) +func TestLiveWritableRemovalClearsMarkerThroughSDK(t *testing.T) { + for _, preUnmount := range []bool{false, true} { + t.Run(map[bool]string{false: "daemon unmount", true: "external unmount"}[preUnmount], func(t *testing.T) { + image := writeVolumeImage(t, "data.img", "original-data") + stack := startTenantStack(t, "node-token", nil, []config.VolumeSpec{{Name: "data", Path: image, Writable: true}}) + writer, err := stack.client.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer writer.Close() + reader, err := stack.client.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer reader.Close() + if _, err = writer.SetVolumes(t.Context(), []sandbox.Volume{{Name: "data", Mode: "rw"}}); err != nil { + t.Fatal(err) + } + if _, err = os.Stat(image + ".dirty"); err != nil { + t.Fatalf("writer has no dirty marker: %v", err) + } + if _, err = reader.SetVolumes(t.Context(), []sandbox.Volume{{Name: "data"}}); err == nil { + t.Fatal("reader admitted while writer was attached") + } + if preUnmount { + stack.eng.mu.Lock() + sock := "" + for guest, mounts := range stack.eng.mounts { + if mounts["/volumes/data"].name == "data" { + sock = guest + break + } + } + stack.eng.mu.Unlock() + if sock == "" { + t.Fatal("writer has no simulated mount") + } + if err = stack.eng.UnmountVolume(t.Context(), sock, "/volumes/data"); err != nil { + t.Fatal(err) + } + } + if _, err = writer.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"data"}}); err != nil { + t.Fatal(err) + } + _, markerErr := os.Stat(image + ".dirty") + if preUnmount && markerErr != nil || !preUnmount && !errors.Is(markerErr, os.ErrNotExist) { + t.Fatalf("dirty marker disagrees with current-call proof: preUnmount=%t err=%v", preUnmount, markerErr) + } + _, err = reader.SetVolumes(t.Context(), []sandbox.Volume{{Name: "data"}}) + if (err != nil) != preUnmount { + t.Fatalf("reader admission after detach: preUnmount=%t err=%v", preUnmount, err) + } + if data, readErr := os.ReadFile(image); readErr != nil || string(data) != "original-data" { + t.Fatalf("detach changed backing contents: %q, %v", data, readErr) + } + }) + } +} + func TestLiveVolumesEndToEnd(t *testing.T) { image := writeVolumeImage(t, "dataset.img", "dataset-bytes") stack := startTenantStack(t, "node-token", nil, []config.VolumeSpec{{Name: "dataset", Path: image}}) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index ec61e655..06a24b05 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -31,7 +31,7 @@ func (e *LiveVolumeUnsupported) Error() string { } type volumeMount struct { - devNum, root, path, options string + devNum, root, path, options, superOptions string } // CheckLiveVolumeSupport rejects runtimes without idempotent live disk attach and detach. @@ -62,7 +62,12 @@ func (e *Engine) MountLiveVolume(ctx context.Context, vsockSocket, name, mount s return e.mountDevice(ctx, vsockSocket, device, name, mount, rw) } -// VolumeDetachResult reports clean-unmount evidence from a confirmed host detach. +// VolumeDetachResult is usable only with a nil DetachVolume error. +// NormalUnmount requires this call to find the expected device at its root +// mount with writable mount and superblock options, ordinarily unmount it, +// exclude remaining device mounts, and confirm host detach. Absence, an earlier +// external unmount, attach-only requests, read-only state, and errors give no +// proof. It cannot be reused across calls or certify externally managed writers. type VolumeDetachResult struct { NormalUnmount bool } @@ -105,8 +110,7 @@ func (e *Engine) unmountLiveVolume(ctx context.Context, vsockSocket, device, mou } return false, err } - mode, _, _ := strings.Cut(atTarget[0].options, ",") - normal = mode == "rw" + normal = writableMountOptions(atTarget[0].options) && writableMountOptions(atTarget[0].superOptions) } if err = e.refuseOtherMounts(ctx, vsockSocket, device); err != nil { return false, err @@ -126,14 +130,23 @@ func (e *Engine) volumeMounts(ctx context.Context, vsockSocket, device string) ( var mounts []volumeMount for line := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") { fields := strings.Fields(line) - if len(fields) < 10 || !slices.Contains(fields[6:], "-") { + if len(fields) < 10 { return "", nil, errors.New("invalid guest mountinfo") } - mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5]}) + separator := slices.Index(fields[6:], "-") + 6 + if separator < 6 || separator+4 != len(fields) { + return "", nil, errors.New("invalid guest mountinfo") + } + mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5], fields[separator+3]}) } return strings.TrimSpace(string(devNum)), mounts, nil } +func writableMountOptions(options string) bool { + parts := strings.Split(options, ",") + return slices.Contains(parts, "rw") && !slices.Contains(parts, "ro") && !slices.Contains(parts, "") +} + func (e *Engine) volumeMounted(ctx context.Context, vsockSocket, device, mount string, rw bool) (bool, error) { devNum, mounts, err := e.volumeMounts(ctx, vsockSocket, device) if err != nil { diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index a3aa2f74..fc1da9c4 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -284,7 +284,7 @@ func guestChanges(calls [][]string) [][]string { } func TestLiveDetachCleanEvidenceRequiresWritableUnmountAndHostSuccess(t *testing.T) { - for _, mode := range []string{"normal", "readonly", "pre-unmounted", "missing device", "host failure", "busy", "residual mount", "attach-only mounted"} { + for _, mode := range []string{"normal", "readonly", "mount readonly", "contradictory mount", "superblock readonly", "contradictory superblock", "empty option", "unordered writable options", "missing superblock options", "extra superblock fields", "pre-unmounted", "missing device", "host failure", "busy", "residual mount", "attach-only mounted"} { t.Run(mode, func(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) @@ -305,6 +305,22 @@ func TestLiveDetachCleanEvidenceRequiresWritableUnmountAndHostSuccess(t *testing fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) case "readonly": fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n") + case "mount readonly": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc rw\n") + case "contradictory mount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw,ro - ext4 /dev/vdc rw\n") + case "empty option": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw,,relatime - ext4 /dev/vdc rw\n") + case "superblock readonly": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw - ext4 /dev/vdc ro,errors=remount-ro\n") + case "contradictory superblock": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw - ext4 /dev/vdc rw,ro\n") + case "unordered writable options": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data relatime,rw shared:4 - ext4 /dev/vdc errors=remount-ro,rw\n") + case "missing superblock options": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw shared:4 - ext4 /dev/vdc\n") + case "extra superblock fields": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw - ext4 /dev/vdc rw extra\n") case "busy": fake.execCode = 32 case "attach-only mounted": @@ -317,8 +333,8 @@ func TestLiveDetachCleanEvidenceRequiresWritableUnmountAndHostSuccess(t *testing } fake.mu.Unlock() result, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, mount) - wantErr := slices.Contains([]string{"host failure", "busy", "residual mount", "attach-only mounted"}, mode) - if (err != nil) != wantErr || result.NormalUnmount != (mode == "normal") { + wantErr := slices.Contains([]string{"host failure", "busy", "residual mount", "attach-only mounted", "missing superblock options", "extra superblock fields"}, mode) + if (err != nil) != wantErr || result.NormalUnmount != (mode == "normal" || mode == "unordered writable options") { t.Fatalf("mode=%s result=%+v err=%v", mode, result, err) } if mode == "attach-only mounted" { diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 9d6bc177..ad8e7482 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -140,7 +140,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ if len(requested) == 0 { return nil, nil } - applied, err := types.ValidateVolumes(requested, false) + applied, err := types.ValidateVolumes(requested) if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } @@ -219,7 +219,7 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ // applyVolume keeps one volume's steps strictly ordered; siblings overlap freely. func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume resolvedVolume, live bool) error { // attach-only mounts nothing, so there is no umount to verify and no marker. - attachOnly := volume.applied.Mount == "" + attachOnly := volume.applied.AttachOnly // write-ahead: the marker must be durable before any guest write can be. if volume.disk.RW && !attachOnly { if err := markVolumeDirty(volume.disk.Path); err != nil { @@ -400,7 +400,7 @@ func writableMounts(volumes []types.Volume) int { } func needsUnmount(volume types.Volume) bool { - return volume.RW() && volume.Mount != "" + return volume.RW() && !volume.AttachOnly } func appliedVolumes(volumes []resolvedVolume) []types.Volume { diff --git a/sandboxd/pool/volume_attach_test.go b/sandboxd/pool/volume_attach_test.go index 4926562f..519f4aff 100644 --- a/sandboxd/pool/volume_attach_test.go +++ b/sandboxd/pool/volume_attach_test.go @@ -376,7 +376,7 @@ func TestQuiesceVolumesSkipsAttachOnlyEntries(t *testing.T) { sb := &types.Sandbox{ ID: "sb_mixed", VMName: "sbx-mixed", Key: testKey, VsockSocket: "/vsock/mixed", Volumes: []types.Volume{ - {Name: "scratch", Mode: types.VolumeModeRW}, + {Name: "scratch", Mode: types.VolumeModeRW, AttachOnly: true}, {Name: "mounted", Mount: "/volumes/mounted", Mode: types.VolumeModeRW}, }, } @@ -402,11 +402,11 @@ func TestWritableMountsCountsOnlyQuiescableEntries(t *testing.T) { }{ {"none", nil, 0}, {"read-only mount", []types.Volume{{Name: "a", Mount: "/a"}}, 0}, - {"attach-only writer", []types.Volume{{Name: "a", Mode: types.VolumeModeRW}}, 0}, + {"attach-only writer", []types.Volume{{Name: "a", Mode: types.VolumeModeRW, AttachOnly: true}}, 0}, {"writable mount", []types.Volume{{Name: "a", Mount: "/a", Mode: types.VolumeModeRW}}, 1}, {"mixed", []types.Volume{ {Name: "a", Mount: "/a", Mode: types.VolumeModeRW}, - {Name: "b", Mode: types.VolumeModeRW}, + {Name: "b", Mode: types.VolumeModeRW, AttachOnly: true}, {Name: "c", Mount: "/c"}, }, 1}, } { diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 2e0e02db..dfe090ae 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -101,7 +101,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = validateLiveTarget(slices.Concat(held, recovery), target); err != nil { return view, err } - additions := volumeDifference(target, held) + additions := volumeDifferenceByName(target, held) if slices.ContainsFunc(additions, func(v types.Volume) bool { return !slices.Contains(recovery, v) }) { if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { return view, err @@ -116,7 +116,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v } sb.Touch() m.mu.Lock() - uncertain := volumeDifference(volumeDifference(sb.VolumesWant, sb.Volumes), target) + uncertain := volumeDifferenceByName(volumeDifferenceByName(sb.VolumesWant, sb.Volumes), target) m.mu.Unlock() for _, v := range uncertain { failedName = v.Name @@ -129,7 +129,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v return view, err } m.mu.Lock() - removals := volumeDifference(sb.Volumes, target) + removals := volumeDifferenceByName(sb.Volumes, target) m.mu.Unlock() for _, v := range removals { failedName = v.Name @@ -142,7 +142,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v m.clearVolumeError(sb, run, v.Name) } m.mu.Lock() - pending := volumeDifference(sb.VolumesWant, sb.Volumes) + pending := volumeDifferenceByName(sb.VolumesWant, sb.Volumes) m.mu.Unlock() for _, v := range pending { failedName = v.Name @@ -201,7 +201,7 @@ func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (contex func (m *Manager) finishVolumeRun(sb *types.Sandbox, run *volumeRun) types.SandboxVolumesResponse { m.mu.Lock() if m.claimed[sb.ID] == sb { - m.releaseVolumes(volumeDifference(run.holds, heldVolumes(sb))) + m.releaseVolumes(volumeDifferenceByName(run.holds, heldVolumes(sb))) } delete(m.volumeRuns, sb) view := m.volumeView(sb) @@ -213,7 +213,7 @@ func (m *Manager) finishVolumeRun(sb *types.Sandbox, run *volumeRun) types.Sandb func (m *Manager) restoreVolumeHolds(sb *types.Sandbox, holds []types.Volume) { if m.volumeRuns[sb] == nil { - m.releaseVolumes(volumeDifference(holds, heldVolumes(sb))) + m.releaseVolumes(volumeDifferenceByName(holds, heldVolumes(sb))) } } @@ -289,7 +289,7 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run } if cleanErr != nil { m.releaseVolumes(additions) - run.holds = volumeDifference(run.holds, additions) + run.holds = volumeDifferenceByName(run.holds, additions) return claimSnapshot{}, false, cleanErr } replaced := func(v types.Volume) bool { @@ -304,13 +304,7 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run } func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume) error { - m.mu.Lock() - err := m.checkVolumeRunLocked(ctx, sb, run) - path := "" - if entry := m.catalog.entries[v.Name]; entry != nil { - path = entry.disk.Path - } - m.mu.Unlock() + path, err := m.detachImagePath(ctx, sb, run, v.Name) if err != nil { return err } @@ -329,6 +323,18 @@ func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *v return nil } +func (m *Manager) detachImagePath(ctx context.Context, sb *types.Sandbox, run *volumeRun, name string) (string, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { + return "", err + } + if entry := m.catalog.entries[name]; entry != nil { + return entry.disk.Path, nil + } + return "", nil +} + func (m *Manager) commitVolumeApplied(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume, attached bool) error { m.mu.Lock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { @@ -357,9 +363,9 @@ func (m *Manager) commitVolumeRun(ctx context.Context, sb *types.Sandbox, run *v if err := ctx.Err(); err != nil { return err } - dropped := volumeDifference(run.holds, heldVolumes(sb)) + dropped := volumeDifferenceByName(run.holds, heldVolumes(sb)) m.releaseVolumes(dropped) - run.holds = volumeDifference(run.holds, dropped) + run.holds = volumeDifferenceByName(run.holds, dropped) return nil } @@ -374,7 +380,7 @@ func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { held := heldVolumes(sb) - recovering := volumeDifference(m.store.recoveryVolumes(sb.ID)[sb.ID], held) + recovering := volumeDifferenceByName(m.store.recoveryVolumes(sb.ID)[sb.ID], held) view := types.SandboxVolumesResponse{ Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), @@ -442,11 +448,11 @@ func (m *Manager) clearVolumeError(sb *types.Sandbox, run *volumeRun, name strin func normalizeLiveVolumes(volumes, held []types.Volume) ([]types.Volume, error) { request := slices.Clone(volumes) for i, v := range request { - if old := volumeIndex(held, v.Name); old >= 0 && held[old].Mount == "" && v.Mount == "" { + if old := volumeIndex(held, v.Name); old >= 0 && held[old].AttachOnly && v.Mount == "" { request[i].AttachOnly = true } } - out, err := types.ValidateVolumes(request, false) + out, err := types.ValidateVolumes(request) if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } @@ -467,7 +473,7 @@ func volumeIndex(volumes []types.Volume, name string) int { return slices.IndexFunc(volumes, func(v types.Volume) bool { return v.Name == name }) } -func volumeDifference(left, right []types.Volume) []types.Volume { +func volumeDifferenceByName(left, right []types.Volume) []types.Volume { return slices.DeleteFunc(slices.Clone(left), func(v types.Volume) bool { return volumeIndex(right, v.Name) >= 0 }) } @@ -484,7 +490,7 @@ func validateDetachNames(names []string) error { } func heldVolumes(sb *types.Sandbox) []types.Volume { - return slices.Concat(sb.Volumes, volumeDifference(sb.VolumesWant, sb.Volumes)) + return slices.Concat(sb.Volumes, volumeDifferenceByName(sb.VolumesWant, sb.Volumes)) } func (m *Manager) resolveLiveAdditions(ctx context.Context, sb *types.Sandbox, additions, recovery []types.Volume) ([]resolvedVolume, error) { diff --git a/sandboxd/server/api.go b/sandboxd/server/api.go index 6e26e0c9..9a26c5df 100644 --- a/sandboxd/server/api.go +++ b/sandboxd/server/api.go @@ -1,6 +1,8 @@ package server import ( + "errors" + "slices" "time" "github.com/cocoonstack/sandbox/sandboxd/types" @@ -42,6 +44,19 @@ func (r ClaimRequest) Key() types.PoolKey { return types.PoolKey{Template: r.Template, Net: r.Net, Size: r.Size}.Defaulted() } +func (r ClaimRequest) normalizedVolumes() ([]types.Volume, error) { + volumes := slices.Clone(r.Volumes) + if r.VolumesAttachOnly { + if len(volumes) == 0 { + return nil, errors.New("volumes_attach_only requires at least one volume") + } + for i := range volumes { + volumes[i].AttachOnly = true + } + } + return types.ValidateVolumes(volumes) +} + // ClaimResponse is the wire reply of POST /v1/claim; success and Redirect are exclusive. type ClaimResponse struct { ID string `json:"id,omitempty"` diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index f9f7ad9a..257b8554 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -358,7 +358,7 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { } func (s *Server) handleVolumeClaim(w http.ResponseWriter, r *http.Request, req ClaimRequest, key types.PoolKey, hash, tenant string) { - volumes, err := types.ValidateVolumes(req.Volumes, req.VolumesAttachOnly) + volumes, err := req.normalizedVolumes() if err != nil { writeErr(w, http.StatusBadRequest, fmt.Sprintf("%v: %v", pool.ErrBadVolume, err)) return diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index 48bf2a94..0704299e 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -22,6 +22,16 @@ import ( const retryAfterSeconds = "10" +// Keep HTTP responses readable by SDKs that echo the historical three-field shape. +// Storage and forwarded requests retain the canonical AttachOnly field. +var responseMarshalers = json.MarshalToFunc(func(out *jsontext.Encoder, v types.Volume) error { + return json.MarshalEncode(out, struct { + Name string `json:"name"` + Mount string `json:"mount,omitempty"` + Mode string `json:"mode,omitempty"` + }{v.Name, v.Mount, v.Mode}) +}) + type tenantKey struct{} func (s *Server) writeRedirect(w http.ResponseWriter, addrs []string) bool { @@ -212,7 +222,7 @@ func noContent(w http.ResponseWriter) func() { func writeJSON(w http.ResponseWriter, code int, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) - _ = json.MarshalWrite(w, v, jsontext.AllowInvalidUTF8(true)) + _ = json.MarshalWrite(w, v, jsontext.AllowInvalidUTF8(true), json.WithMarshalers(responseMarshalers)) } func writeErr(w http.ResponseWriter, code int, msg string) { diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 98d4756d..a51dce7e 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -174,11 +174,13 @@ func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { func TestLiveVolumeRoutesPreserveMixedAttachOnlyRoundtrip(t *testing.T) { want := []types.Volume{{Name: "mounted", Mount: "/volumes/mounted"}, {Name: "raw", Mode: types.VolumeModeRW, AttachOnly: true}} + legacy := slices.Clone(want) + legacy[1].AttachOnly = false view := types.SandboxVolumesResponse{VolumesWant: want, Volumes: want} mgr := &fakeManager{ getVolumes: func(string, string) (types.SandboxVolumesResponse, error) { return view, nil }, setVolumes: func(_, _ string, got []types.Volume) (types.SandboxVolumesResponse, error) { - if !slices.Equal(got, want) { + if !slices.Equal(got, legacy) { t.Errorf("GET-to-PUT changed mixed intent: %+v", got) } return view, nil @@ -194,15 +196,19 @@ func TestLiveVolumeRoutesPreserveMixedAttachOnlyRoundtrip(t *testing.T) { get := doReq(t, http.MethodGet, ts.URL+"/v1/sandboxes/sb_1/volumes", "root", "") defer get.Body.Close() var got types.SandboxVolumesResponse - if err := json.UnmarshalRead(get.Body, &got); err != nil || !slices.Equal(got.VolumesWant, want) { - t.Fatalf("GET lost explicit attach-only flag: %+v, %v", got, err) + if err := json.UnmarshalRead(get.Body, &got); err != nil || !slices.Equal(got.VolumesWant, legacy) { + t.Fatalf("GET broke the published response shape: %+v, %v", got, err) } for _, method := range []string{http.MethodPut, http.MethodPatch} { key := "volumes" if method == http.MethodPatch { key = "add" } - body, err := json.Marshal(map[string][]types.Volume{key: got.VolumesWant}) + input := got.VolumesWant + if method == http.MethodPatch { + input = want + } + body, err := json.Marshal(map[string][]types.Volume{key: input}) if err != nil { t.Fatal(err) } diff --git a/sandboxd/server/volume_wire_test.go b/sandboxd/server/volume_wire_test.go new file mode 100644 index 00000000..a1e52304 --- /dev/null +++ b/sandboxd/server/volume_wire_test.go @@ -0,0 +1,127 @@ +package server + +import ( + "context" + "encoding/json/jsontext" + "encoding/json/v2" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestClaimVolumeCompatibilityExpansion(t *testing.T) { + for _, tt := range []struct { + name string + req ClaimRequest + want []types.Volume + err bool + }{ + {"default mount", ClaimRequest{Volumes: []types.Volume{{Name: "data"}}}, []types.Volume{{Name: "data", Mount: "/volumes/data"}}, false}, + {"legacy all devices", ClaimRequest{VolumesAttachOnly: true, Volumes: []types.Volume{{Name: "data"}, {Name: "work", Mode: types.VolumeModeRW}}}, []types.Volume{{Name: "data", AttachOnly: true}, {Name: "work", Mode: types.VolumeModeRW, AttachOnly: true}}, false}, + {"mixed", ClaimRequest{Volumes: []types.Volume{{Name: "data"}, {Name: "work", AttachOnly: true}}}, []types.Volume{{Name: "data", Mount: "/volumes/data"}, {Name: "work", AttachOnly: true}}, false}, + {"mixed reversed", ClaimRequest{Volumes: []types.Volume{{Name: "work", AttachOnly: true}, {Name: "data"}}}, []types.Volume{{Name: "work", AttachOnly: true}, {Name: "data", Mount: "/volumes/data"}}, false}, + {"empty legacy", ClaimRequest{VolumesAttachOnly: true}, nil, true}, + {"legacy mount conflict", ClaimRequest{VolumesAttachOnly: true, Volumes: []types.Volume{{Name: "data", Mount: "/data"}}}, nil, true}, + } { + t.Run(tt.name, func(t *testing.T) { + before := slices.Clone(tt.req.Volumes) + got, err := tt.req.normalizedVolumes() + if (err != nil) != tt.err || !slices.Equal(got, tt.want) { + t.Fatalf("normalized volumes=%+v err=%v, want %+v error=%t", got, err, tt.want, tt.err) + } + if !slices.Equal(tt.req.Volumes, before) { + t.Fatal("legacy expansion mutated caller input") + } + }) + } +} + +func TestVolumeHTTPResponsesPreservePublishedWireShape(t *testing.T) { + volumes := []types.Volume{{Name: "raw", Mode: types.VolumeModeRW, AttachOnly: true}, {Name: "mounted", Mount: "/data"}} + view := types.SandboxVolumesResponse{Volumes: volumes, VolumesWant: volumes} + summary := pool.SandboxSummary{Volumes: volumes, VolumesWant: volumes} + for name, payload := range map[string]any{ + "claim": ClaimResponse{Volumes: volumes}, + "fork": ForkResponse{Children: []ClaimResponse{{Volumes: volumes}}}, + "volumes": view, + "error": types.SandboxVolumesResponse{Volumes: volumes, VolumesWant: volumes, Error: "retry"}, + "summary": summary, + "list": map[string]any{"sandboxes": []*pool.SandboxSummary{&summary}}, + "pointer": &volumes[0], + } { + t.Run(name, func(t *testing.T) { + reply := httptest.NewRecorder() + writeJSON(reply, http.StatusOK, payload) + body := reply.Body.String() + if !jsontext.Value(body).IsValid() || strings.Contains(body, "attach_only") || !strings.Contains(body, `"name":"raw","mode":"rw"`) { + t.Fatalf("published response shape changed: %s", body) + } + }) + } + for _, payload := range []any{ClaimRequest{Volumes: volumes}, &types.Sandbox{Volumes: volumes, VolumesWant: volumes}} { + raw, err := json.Marshal(payload) + if err != nil || !strings.Contains(string(raw), `"attach_only":true`) { + t.Fatalf("internal serialization lost attachment intent: %s, %v", raw, err) + } + } +} + +func TestVolumeHTTPProjectionPreservesInvalidUTF8Handling(t *testing.T) { + reply := httptest.NewRecorder() + writeJSON(reply, http.StatusOK, []types.Volume{{Name: "raw\xff", AttachOnly: true}}) + if got := reply.Body.String(); !jsontext.Value(got).IsValid() || !strings.Contains(got, "raw�") { + t.Fatalf("response no longer replaces invalid UTF8: %q", got) + } +} + +func TestVolumeRoutesKeepLegacyResponseFields(t *testing.T) { + volumes := []types.Volume{{Name: "raw", Mode: types.VolumeModeRW, AttachOnly: true}} + sb := &types.Sandbox{ID: "sb_1", Token: "tok", Volumes: volumes} + view := types.SandboxVolumesResponse{Volumes: volumes, VolumesWant: volumes} + summary := pool.SandboxSummary{ID: sb.ID, Volumes: volumes, VolumesWant: volumes} + mgr := &fakeManager{ + claim: func(context.Context, types.PoolKey, time.Duration) (*types.Sandbox, error) { return sb, nil }, + fork: func(string, string, int, time.Duration) ([]*types.Sandbox, error) { return []*types.Sandbox{sb}, nil }, + claimCheckpoint: func(string) (*types.Sandbox, error) { return sb, nil }, + sandboxByID: func(string) (pool.SandboxSummary, bool) { return summary, true }, + sandboxIndex: func(string, string) []pool.SandboxSummary { return []pool.SandboxSummary{summary} }, + getVolumes: func(string, string) (types.SandboxVolumesResponse, error) { return view, nil }, + setVolumes: func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) { return view, nil }, + patchVolumes: func(string, string, types.VolumePatch) (types.SandboxVolumesResponse, error) { + return view, pool.ErrBadVolume + }, + } + ts := newTestServer(t, "root", mgr, nil) + for _, tt := range []struct { + method, path, body string + status int + }{ + {http.MethodPost, "/v1/claim", `{"template":"rt:24.04"}`, http.StatusOK}, + {http.MethodPost, "/v1/sandboxes/sb_1/fork", `{"token":"tok","count":1}`, http.StatusOK}, + {http.MethodPost, "/v1/checkpoints/ckpt_1/claim", `{}`, http.StatusOK}, + {http.MethodGet, "/v1/sandboxes/sb_1", "", http.StatusOK}, + {http.MethodGet, "/v1/sandboxes", "", http.StatusOK}, + {http.MethodGet, "/v1/sandboxes/sb_1/volumes", "", http.StatusOK}, + {http.MethodPut, "/v1/sandboxes/sb_1/volumes", `{"volumes":[]}`, http.StatusOK}, + {http.MethodPatch, "/v1/sandboxes/sb_1/volumes", `{"remove":["raw"]}`, http.StatusBadRequest}, + } { + t.Run(tt.method+tt.path, func(t *testing.T) { + reply := doReq(t, tt.method, ts.URL+tt.path, "root", tt.body) + defer reply.Body.Close() + var body map[string]jsontext.Value + if err := json.UnmarshalRead(reply.Body, &body); err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(body) + if err != nil || reply.StatusCode != tt.status || strings.Contains(string(raw), "attach_only") || !strings.Contains(string(raw), `"name":"raw","mode":"rw"`) { + t.Fatalf("legacy response changed: status=%d body=%s err=%v", reply.StatusCode, raw, err) + } + }) + } +} diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 523c354c..1c537ead 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -804,13 +804,10 @@ func VolumeRWNames(volumes []Volume) []string { } // ValidateVolumes returns detached entries with defaults filled and modes normalized. -func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { +func ValidateVolumes(volumes []Volume) ([]Volume, error) { if len(volumes) > MaxClaimVolumes { return nil, fmt.Errorf("volumes must contain at most %d entries, got %d", MaxClaimVolumes, len(volumes)) } - if attachOnly && len(volumes) == 0 { - return nil, errors.New("volumes_attach_only requires at least one volume") - } applied := make([]Volume, len(volumes)) names := make(map[string]struct{}, len(volumes)) for i, volume := range volumes { @@ -829,7 +826,7 @@ func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { default: return nil, fmt.Errorf("volumes[%d] mode %q must be %s or %s", i, mode, VolumeModeRO, VolumeModeRW) } - if attachOnly || volume.AttachOnly { + if volume.AttachOnly { if volume.Mount != "" { return nil, fmt.Errorf("volumes[%d] mount %q is meaningless when the caller mounts the device itself", i, volume.Mount) } diff --git a/sandboxd/types/volume_test.go b/sandboxd/types/volume_test.go index f1c2dbbe..c7f428f1 100644 --- a/sandboxd/types/volume_test.go +++ b/sandboxd/types/volume_test.go @@ -31,7 +31,7 @@ func TestValidVolumeName(t *testing.T) { func TestValidateVolumes(t *testing.T) { volumes := []Volume{{Name: "dataset"}, {Name: "weights-1", Mount: "/models"}} wantInput := slices.Clone(volumes) - got, err := ValidateVolumes(volumes, false) + got, err := ValidateVolumes(volumes) if err != nil { t.Fatalf("ValidateVolumes: %v", err) } @@ -54,7 +54,7 @@ func TestValidateVolumes(t *testing.T) { {Name: "f"}, {Name: "g"}, {Name: "h"}, - }, false); err != nil || len(got) != MaxClaimVolumes { + }); err != nil || len(got) != MaxClaimVolumes { t.Errorf("ValidateVolumes at limit: got=%v err=%v", got, err) } @@ -75,7 +75,7 @@ func TestValidateVolumes(t *testing.T) { {"uppercase-mode", []Volume{{Name: "dataset", Mode: "RW"}}}, } { t.Run(tt.name, func(t *testing.T) { - if _, err := ValidateVolumes(tt.volumes, false); err == nil { + if _, err := ValidateVolumes(tt.volumes); err == nil { t.Fatal("ValidateVolumes succeeded") } }) @@ -87,7 +87,7 @@ func TestValidateVolumesNormalizesMode(t *testing.T) { {Name: "shared"}, {Name: "explicit", Mode: VolumeModeRO}, {Name: "writable", Mode: VolumeModeRW}, - }, false) + }) if err != nil { t.Fatalf("ValidateVolumes: %v", err) } @@ -112,10 +112,10 @@ func TestValidateVolumesNormalizesMode(t *testing.T) { func TestValidateVolumesAttachOnly(t *testing.T) { got, err := ValidateVolumes([]Volume{ - {Name: "dataset"}, - {Name: "scratch", Mode: VolumeModeRW}, - {Name: "explicit", Mode: VolumeModeRO}, - }, true) + {Name: "dataset", AttachOnly: true}, + {Name: "scratch", Mode: VolumeModeRW, AttachOnly: true}, + {Name: "explicit", Mode: VolumeModeRO, AttachOnly: true}, + }) if err != nil { t.Fatalf("ValidateVolumes: %v", err) } @@ -132,7 +132,6 @@ func TestValidateVolumesAttachOnly(t *testing.T) { name string volumes []Volume }{ - {"no volumes", nil}, {"default mount spelled out", []Volume{{Name: "dataset", Mount: "/volumes/dataset"}}}, {"custom mount", []Volume{{Name: "dataset", Mount: "/datasets"}}}, {"one entry of many", []Volume{{Name: "dataset"}, {Name: "scratch", Mount: "/scratch"}}}, @@ -141,7 +140,10 @@ func TestValidateVolumesAttachOnly(t *testing.T) { {"unknown mode", []Volume{{Name: "dataset", Mode: "readwrite"}}}, } { t.Run(tt.name, func(t *testing.T) { - if _, err := ValidateVolumes(tt.volumes, true); err == nil { + for i := range tt.volumes { + tt.volumes[i].AttachOnly = true + } + if _, err := ValidateVolumes(tt.volumes); err == nil { t.Fatal("ValidateVolumes succeeded") } }) @@ -151,18 +153,18 @@ func TestValidateVolumesAttachOnly(t *testing.T) { func TestValidateVolumesRejectsGuestOSMounts(t *testing.T) { for _, root := range append([]string{"/"}, guestOSMountRoots...) { t.Run(root, func(t *testing.T) { - if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root}}, false); err == nil { + if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root}}); err == nil { t.Fatalf("accepted OS mount %q", root) } if root != "/" { - if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root + "/child"}}, false); err == nil { + if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root + "/child"}}); err == nil { t.Fatalf("accepted mount under OS root %q", root) } } }) } for _, mount := range []string{"/data", "/home/dataset", "/opt-data", "/usrdata"} { - if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: mount}}, false); err != nil { + if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: mount}}); err != nil { t.Errorf("rejected allowed mount %q: %v", mount, err) } } @@ -170,7 +172,7 @@ func TestValidateVolumesRejectsGuestOSMounts(t *testing.T) { func TestValidateMixedVolumesPreservesPerEntryIntent(t *testing.T) { request := []Volume{{Name: "mounted"}, {Name: "writer", Mode: VolumeModeRW}, {Name: "raw", AttachOnly: true}, {Name: "raw-writer", Mode: VolumeModeRW, AttachOnly: true}} - got, err := ValidateVolumes(request, false) + got, err := ValidateVolumes(request) if err != nil { t.Fatal(err) } @@ -178,11 +180,11 @@ func TestValidateMixedVolumesPreservesPerEntryIntent(t *testing.T) { if !slices.Equal(got, want) { t.Fatalf("mixed intent changed: got %+v, want %+v", got, want) } - again, err := ValidateVolumes(got, false) + again, err := ValidateVolumes(got) if err != nil || !slices.Equal(again, want) { t.Fatalf("second validation changed mixed intent: %+v, %v", again, err) } - if _, err = ValidateVolumes([]Volume{{Name: "raw", AttachOnly: true, Mount: "/data"}}, false); err == nil { + if _, err = ValidateVolumes([]Volume{{Name: "raw", AttachOnly: true, Mount: "/data"}}); err == nil { t.Fatal("accepted attach-only with a mount") } } From e6e474e330574247a5cd8ec9d6fab783ac6e0996 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 03:20:21 +0000 Subject: [PATCH 21/72] sdk: Preserve attach-only intent across server versions --- sdk/go/client.go | 30 ++++++--- sdk/go/info.go | 4 ++ sdk/go/template.go | 2 +- sdk/go/volume_mutation_test.go | 57 +++++++++++++++- sdk/go/volumes.go | 32 ++++++++- sdk/go/volumes_test.go | 86 +++++++++++++++--------- sdk/python/cocoonsandbox/client.py | 13 ++-- sdk/python/cocoonsandbox/sandbox.py | 6 +- sdk/python/cocoonsandbox/template.py | 19 ++++-- sdk/python/cocoonsandbox/volumes.py | 23 +++++-- sdk/python/tests/test_client.py | 97 +++++++++++++++++++++------- sdk/python/tests/test_volumes.py | 44 +++++++++++-- 12 files changed, 326 insertions(+), 87 deletions(-) diff --git a/sdk/go/client.go b/sdk/go/client.go index 582d19d4..70bf2632 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -110,17 +110,30 @@ func (r claimRequest) rejectPinnedAxes() error { return nil } -func (r claimRequest) validateVolumes() error { +func (r *claimRequest) prepareVolumes() error { + attachOnly := 0 for _, v := range r.Volumes { - if v.Mode != "" && v.Mode != volumeModeRW { - return fmt.Errorf("volume %q: mode must be \"\", %q, or %q, got %q", v.Name, volumeModeRO, volumeModeRW, v.Mode) - } - if v.AttachOnly && v.Mount != "" { - return fmt.Errorf("volume %q: mount %q is meaningless with attach_only, which leaves mounting to the caller", v.Name, v.Mount) + if err := validateVolume(v); err != nil { + return err } if r.VolumesAttachOnly && v.Mount != "" { return fmt.Errorf("volume %q: mount %q is meaningless with WithVolumesAttachOnly, which leaves mounting to the caller", v.Name, v.Mount) } + if v.AttachOnly { + attachOnly++ + } + } + if !r.VolumesAttachOnly && attachOnly > 0 && attachOnly != len(r.Volumes) { + return errors.New("mixed mounted and attach-only initial claims are not safe on older servers: claim without volumes, then use SetVolumes or PatchVolumes") + } + if attachOnly > 0 { + r.VolumesAttachOnly = true + } + if r.VolumesAttachOnly { + r.Volumes = slices.Clone(r.Volumes) + for i := range r.Volumes { + r.Volumes[i].AttachOnly = false + } } return nil } @@ -175,7 +188,7 @@ func (c *Client) New(ctx context.Context, template string, opts ...Option) (*San for _, opt := range opts { opt(&claim) } - if err := claim.validateVolumes(); err != nil { + if err := claim.prepareVolumes(); err != nil { return nil, err } addr, cr, err := claimFollow(c.addr, "claim", func(noRedirect, requirePromoted bool) ([]byte, error) { @@ -253,8 +266,9 @@ func (c *Client) ownerAt(ctx context.Context, addr, id, token string) (string, e } func (c *Client) handleFrom(dialed string, cr claimResponse) *Sandbox { + volumes := responseVolumes(cr.Volumes) return &Sandbox{ - ID: cr.ID, Deadline: cr.Deadline, Volumes: cr.Volumes, VolumesWant: slices.Clone(cr.Volumes), + ID: cr.ID, Deadline: cr.Deadline, Volumes: volumes, VolumesWant: slices.Clone(volumes), FromCheckpoint: cr.FromCheckpoint, TemplateDigest: cr.TemplateDigest, NetRoute: cr.NetRoute, c: c, token: cr.Token, owner: cmp.Or(cr.OwnerAddr, dialed), } diff --git a/sdk/go/info.go b/sdk/go/info.go index 86879cd3..3a08e819 100644 --- a/sdk/go/info.go +++ b/sdk/go/info.go @@ -90,6 +90,10 @@ func (c *Client) Sandboxes(ctx context.Context) ([]SandboxSummary, error) { if err != nil { return nil, err } + for i := range reply.Sandboxes { + reply.Sandboxes[i].Volumes = responseVolumes(reply.Sandboxes[i].Volumes) + reply.Sandboxes[i].VolumesWant = responseVolumes(reply.Sandboxes[i].VolumesWant) + } return reply.Sandboxes, nil } diff --git a/sdk/go/template.go b/sdk/go/template.go index 34ee31f7..e9e08a9a 100644 --- a/sdk/go/template.go +++ b/sdk/go/template.go @@ -29,7 +29,7 @@ func (t *Template) New(ctx context.Context, opts ...Option) (*Sandbox, error) { if err := claim.rejectPinnedAxes(); err != nil { return nil, err } - if err := claim.validateVolumes(); err != nil { + if err := claim.prepareVolumes(); err != nil { return nil, err } claim.Net, claim.Size = t.net, t.size diff --git a/sdk/go/volume_mutation_test.go b/sdk/go/volume_mutation_test.go index 44fc937b..3b06f079 100644 --- a/sdk/go/volume_mutation_test.go +++ b/sdk/go/volume_mutation_test.go @@ -51,7 +51,7 @@ func TestLiveVolumesUseOwnerAndAPIToken(t *testing.T) { func TestLiveVolumeFailureRetainsPartialView(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) - _, _ = io.WriteString(w, `{"volumes_want":[{"name":"new","mode":"rw","attach_only":true}],"volumes":[{"name":"old","attach_only":true}],"converging":false,"states":[{"name":"new","state":"pending","error":"retry"},{"name":"old","state":"detaching"}],"error":"convergence failed"}`) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"new","mode":"rw"}],"volumes":[{"name":"old"}],"converging":false,"states":[{"name":"new","state":"pending","error":"retry"},{"name":"old","state":"detaching"}],"error":"convergence failed"}`) })) t.Cleanup(ts.Close) c := testClient(t, ts) @@ -230,3 +230,58 @@ func TestLiveVolumeAttachOnlyRoundTrip(t *testing.T) { t.Fatal("roundtrip changed handle state") } } + +func TestLiveVolumesRejectConflictingAttachOnlyBeforeRequest(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("invalid volume reached server") })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + sb := c.Attach(c.addr, "sb_1", "tok") + volumes := []Volume{{Name: "data", Mount: "/data", AttachOnly: true}} + if _, err := sb.SetVolumes(t.Context(), volumes); err == nil { + t.Fatal("SetVolumes accepted mount with attach_only") + } + if _, err := sb.PatchVolumes(t.Context(), VolumePatch{Add: volumes}); err == nil { + t.Fatal("PatchVolumes accepted mount with attach_only") + } +} + +func TestVolumeResponseInferenceDoesNotChangeRequestDecoding(t *testing.T) { + var request Volume + if err := json.Unmarshal([]byte(`{"name":"default-mounted"}`), &request); err != nil || request.AttachOnly { + t.Fatalf("ordinary request decode acquired raw intent: %+v err=%v", request, err) + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/sandboxes" { + _, _ = io.WriteString(w, `{"sandboxes":[{"id":"sb_1","volumes":[{"name":"raw"}],"volumes_want":[{"name":"mounted","mount":"/data"}]},{"id":"sb_2","volumes":[],"volumes_want":[]},{"id":"sb_3"}]}`) + return + } + _, _ = io.WriteString(w, `{"volumes":[{"name":"raw"}],"volumes_want":[{"name":"raw","mount":""}],"states":[],"converging":false}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + rows, err := c.Sandboxes(t.Context()) + if err != nil || !rows[0].Volumes[0].AttachOnly || rows[0].VolumesWant[0].AttachOnly || rows[1].Volumes == nil || rows[2].Volumes != nil { + t.Fatalf("summary inference or nil/empty changed: %+v err=%v", rows, err) + } + sb := c.Attach(c.addr, "sb_1", "tok") + view, err := sb.GetVolumes(t.Context()) + if err != nil || !view.Volumes[0].AttachOnly || !view.VolumesWant[0].AttachOnly { + t.Fatalf("live inference: %+v err=%v", view, err) + } +} + +func TestClaimResponseVolumesAreOwnedCopies(t *testing.T) { + c := &Client{} + wire := []Volume{{Name: "raw"}, {Name: "mounted", Mount: "/data"}} + sb := c.handleFrom("node", claimResponse{Volumes: wire}) + if !sb.Volumes[0].AttachOnly || sb.Volumes[1].AttachOnly || wire[0].AttachOnly { + t.Fatalf("response inference changed wire data: handle=%+v wire=%+v", sb.Volumes, wire) + } + sb.Volumes[0].Name = "caller" + if wire[0].Name != "raw" || sb.VolumesWant[0].Name != "raw" { + t.Fatal("claim volume views alias the wire or each other") + } + if c.handleFrom("node", claimResponse{}).Volumes != nil || c.handleFrom("node", claimResponse{Volumes: []Volume{}}).Volumes == nil { + t.Fatal("claim response changed nil versus empty volume semantics") + } +} diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go index 0a4917fe..24b0310f 100644 --- a/sdk/go/volumes.go +++ b/sdk/go/volumes.go @@ -60,6 +60,11 @@ func (s *Sandbox) SetVolumes(ctx context.Context, volumes []Volume) (SandboxVolu if volumes == nil { return SandboxVolumesResponse{}, errors.New("set volumes: use an explicit empty slice to clear volumes") } + for _, volume := range volumes { + if err := validateVolume(volume); err != nil { + return SandboxVolumesResponse{}, err + } + } return s.requestVolumes(ctx, http.MethodPut, struct { Volumes []Volume `json:"volumes"` }{Volumes: volumes}, "set volumes") @@ -67,9 +72,34 @@ func (s *Sandbox) SetVolumes(ctx context.Context, volumes []Volume) (SandboxVolu // PatchVolumes adds or removes catalog volumes by name. func (s *Sandbox) PatchVolumes(ctx context.Context, patch VolumePatch) (SandboxVolumesResponse, error) { + for _, volume := range patch.Add { + if err := validateVolume(volume); err != nil { + return SandboxVolumesResponse{}, err + } + } return s.requestVolumes(ctx, http.MethodPatch, patch, "patch volumes") } +func validateVolume(v Volume) error { + if v.Mode != "" && v.Mode != volumeModeRO && v.Mode != volumeModeRW { + return fmt.Errorf("volume %q: mode must be \"\", %q, or %q, got %q", v.Name, volumeModeRO, volumeModeRW, v.Mode) + } + if v.AttachOnly && v.Mount != "" { + return fmt.Errorf("volume %q: mount %q is meaningless with attach_only, which leaves mounting to the caller", v.Name, v.Mount) + } + return nil +} + +// Historical responses identify attach-only devices by their absent mount. +// Keep that inference out of request decoding, where an absent mount means the default path. +func responseVolumes(volumes []Volume) []Volume { + volumes = slices.Clone(volumes) + for i := range volumes { + volumes[i].AttachOnly = volumes[i].Mount == "" + } + return volumes +} + func (s *Sandbox) requestVolumes(ctx context.Context, method string, request any, verb string) (SandboxVolumesResponse, error) { var out SandboxVolumesResponse var body io.Reader @@ -95,7 +125,7 @@ func (s *Sandbox) requestVolumes(ctx context.Context, method string, request any decodeErr := json.UnmarshalRead(resp.Body, &wire) complete := decodeErr == nil && wire.VolumesWant != nil && wire.Volumes != nil && wire.States != nil && wire.Converging != nil if complete { - out = SandboxVolumesResponse{VolumesWant: *wire.VolumesWant, Volumes: *wire.Volumes, States: *wire.States, Converging: *wire.Converging, Error: wire.Error} + out = SandboxVolumesResponse{VolumesWant: responseVolumes(*wire.VolumesWant), Volumes: responseVolumes(*wire.Volumes), States: *wire.States, Converging: *wire.Converging, Error: wire.Error} s.VolumesWant = slices.Clone(out.VolumesWant) s.Volumes = slices.Clone(out.Volumes) } diff --git a/sdk/go/volumes_test.go b/sdk/go/volumes_test.go index c21ce4fa..b4a4da9d 100644 --- a/sdk/go/volumes_test.go +++ b/sdk/go/volumes_test.go @@ -254,7 +254,11 @@ func TestWithVolumesAttachOnlySendsFlagAndDecodesMountlessEcho(t *testing.T) { if mount, present := entry["mount"]; present { t.Errorf("volume %v carries mount %v, want none", entry["name"], mount) } + if _, present := entry["attach_only"]; present { + t.Error("legacy attach-only claim carries a per-entry flag") + } } + echo[0].AttachOnly, echo[1].AttachOnly = true, true if !slices.Equal(sb.Volumes, echo) { t.Errorf("volumes = %+v, want %+v", sb.Volumes, echo) } @@ -302,7 +306,7 @@ func TestRejectsInvalidVolumeModeLocally(t *testing.T) { } func TestSandboxVolumesEchoMode(t *testing.T) { - want := []Volume{{Name: "imagenet"}, {Name: "scratch", Mode: "rw"}} + want := []Volume{{Name: "imagenet", Mount: "/volumes/imagenet"}, {Name: "scratch", Mount: "/volumes/scratch", Mode: "rw"}} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, claimResponse{ID: "sb_1", Token: "tok", Volumes: want}) })) @@ -317,39 +321,57 @@ func TestSandboxVolumesEchoMode(t *testing.T) { } } -func TestPerEntryAttachOnlyClaimModes(t *testing.T) { +func TestPerEntryAttachOnlyClaimCompatibility(t *testing.T) { for _, promoted := range []bool{false, true} { - t.Run(fmt.Sprint(promoted), func(t *testing.T) { - want := []Volume{ - {Name: "mounted-ro"}, - {Name: "mounted-rw", Mount: "/work", Mode: "rw"}, - {Name: "device-ro", AttachOnly: true}, - {Name: "device-rw", Mode: "rw", AttachOnly: true}, - } - var got claimRequest - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.UnmarshalRead(r.Body, &got); err != nil { - t.Errorf("decode mixed claim: %v", err) + for _, mixed := range []bool{false, true} { + t.Run(fmt.Sprintf("promoted=%t/mixed=%t", promoted, mixed), func(t *testing.T) { + volumes := []Volume{{Name: "device-ro", AttachOnly: true}, {Name: "device-rw", Mode: "rw", AttachOnly: true}} + if mixed { + volumes = append(volumes, Volume{Name: "mounted-ro"}) } - _ = json.MarshalWrite(w, claimResponse{ID: "sb_1", Token: "tok", Volumes: want}) - })) - t.Cleanup(ts.Close) - c := testClient(t, ts) - var sb *Sandbox - var err error - if promoted { - tpl := &Template{Name: "task:v1", c: c, addr: c.addr, net: "none", size: "small"} - sb, err = tpl.New(t.Context(), WithVolumes(want...)) - } else { - sb, err = c.New(t.Context(), "rt:24.04", WithVolumes(want...)) - } - if err != nil { - t.Fatal(err) - } - if got.VolumesAttachOnly || !slices.Equal(got.Volumes, want) || !slices.Equal(sb.Volumes, want) { - t.Fatalf("mixed claim=%+v reply=%+v, want=%+v", got, sb.Volumes, want) - } - }) + var got claimRequest + requests := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if err := json.UnmarshalRead(r.Body, &got); err != nil { + t.Errorf("decode claim: %v", err) + } + // An older server understands only the claim-wide flag. + _ = json.MarshalWrite(w, claimResponse{ID: "sb_1", Token: "tok", Volumes: got.Volumes}) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + option := WithVolumes(volumes...) + var sb *Sandbox + var err error + if promoted { + tpl := &Template{Name: "task:v1", c: c, addr: c.addr, net: "none", size: "small"} + sb, err = tpl.New(t.Context(), option) + } else { + sb, err = c.New(t.Context(), "rt:24.04", option) + } + if mixed { + if err == nil || !strings.Contains(err.Error(), "claim without volumes") || requests != 0 { + t.Fatalf("mixed claim reached server: requests=%d err=%v", requests, err) + } + return + } + if err != nil { + t.Fatal(err) + } + if !got.VolumesAttachOnly || !slices.Equal(sb.Volumes, volumes) { + t.Fatalf("claim=%+v reply=%+v want=%+v", got, sb.Volumes, volumes) + } + for _, v := range got.Volumes { + if v.AttachOnly { + t.Fatal("claim retained per-entry attach_only") + } + } + if _, err = c.New(t.Context(), "rt:24.04", option); err != nil || !got.VolumesAttachOnly { + t.Fatalf("reused option lost raw intent: claim=%+v err=%v", got, err) + } + }) + } } } diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 74ce4174..4e8be349 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -18,7 +18,7 @@ from .errors import APIError, SandboxTimeout, require_field from .sandbox import Sandbox from .template import _claim_body, _template_query -from .volumes import Volume +from .volumes import Volume, _response_volumes T = TypeVar("T") @@ -61,7 +61,7 @@ def new( size: str = "", ttl_seconds: int = 0, claim_ref: str = "", - volumes: Sequence[str | Mapping[str, str | bool] | Volume] | None = None, + volumes: Iterable[str | Mapping[str, str | bool] | Volume] | None = None, mount: bool = True, *, deadline: float | None = None, @@ -110,7 +110,12 @@ def attach(self, owner_addr: str, id: str, token: str) -> Sandbox: def sandboxes(self) -> list[dict[str, Any]]: """Lists the claims this token may see: id, key, deadline, claim_ref — never tokens or host paths.""" reply = self._request(self.addr, "GET", "/v1/sandboxes", None, "list sandboxes") - return [dict(sb) for sb in reply.get("sandboxes") or []] + rows = [dict(sb) for sb in reply.get("sandboxes") or []] + for row in rows: + for key in ("volumes", "volumes_want"): + if row.get(key) is not None: + row[key] = _response_volumes(row[key]) + return rows def drain(self) -> dict[str, Any]: """Cordons the node (root token): new claims are refused, live ones run to their leases.""" @@ -181,7 +186,7 @@ def _handle_from(self, dialed: str, reply: dict[str, Any], verb: str = "claim") deadline=reply.get("deadline", ""), from_checkpoint=reply.get("from_checkpoint", ""), template_digest=reply.get("template_digest", ""), - volumes=reply.get("volumes") or [], + volumes=_response_volumes(reply.get("volumes") or []), ) def _post_json( diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 46666d34..4981a222 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -7,7 +7,7 @@ import threading import time from collections import deque -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, cast from .checkpoint import Checkpoint @@ -74,7 +74,7 @@ def get_volumes(self, *, timeout: float | None = None) -> SandboxVolumesResponse return self._volumes_request("GET", None, "get volumes", timeout) def set_volumes( - self, volumes: Sequence[str | Mapping[str, str | bool] | Volume], *, timeout: float | None = None + self, volumes: Iterable[str | Mapping[str, str | bool] | Volume], *, timeout: float | None = None ) -> SandboxVolumesResponse: """Replaces the desired set and converges; an explicit empty list clears volumes.""" if volumes is None: @@ -84,7 +84,7 @@ def set_volumes( def patch_volumes( self, *, - add: Sequence[str | Mapping[str, str | bool] | Volume] | None = None, + add: Iterable[str | Mapping[str, str | bool] | Volume] | None = None, remove: list[str] | None = None, timeout: float | None = None, ) -> SandboxVolumesResponse: diff --git a/sdk/python/cocoonsandbox/template.py b/sdk/python/cocoonsandbox/template.py index 56a0ea05..9e809e81 100644 --- a/sdk/python/cocoonsandbox/template.py +++ b/sdk/python/cocoonsandbox/template.py @@ -3,7 +3,7 @@ from __future__ import annotations import urllib.parse -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any from .volumes import Volume, _volume_bodies @@ -27,7 +27,7 @@ def __init__(self, client: Client, addr: str, name: str, net: str, size: str, co def new( self, ttl_seconds: int = 0, - volumes: Sequence[str | Mapping[str, str | bool] | Volume] | None = None, + volumes: Iterable[str | Mapping[str, str | bool] | Volume] | None = None, mount: bool = True, *, deadline: float | None = None, @@ -36,7 +36,7 @@ def new( mount=False attaches the volumes without mounting them.""" claim = _claim_body(self.name, self.net, self.size, ttl_seconds, volumes, mount) claim["require_promoted"] = True - if volumes: + if claim.get("volumes"): return self._client._claim_from(self._addr, claim, deadline=deadline) claim["no_redirect"] = True reply = self._client._post_json(self._addr, "/v1/claim", claim, "claim", deadline=deadline) @@ -55,7 +55,7 @@ def _claim_body( net: str, size: str, ttl_seconds: int, - volumes: Sequence[str | Mapping[str, str | bool] | Volume] | None = None, + volumes: Iterable[str | Mapping[str, str | bool] | Volume] | None = None, mount: bool = True, claim_ref: str = "", ) -> dict[str, Any]: @@ -68,6 +68,17 @@ def _claim_body( claim["ttl_seconds"] = ttl_seconds if volumes is not None: entries = _volume_bodies(volumes, mount) + raw = sum(bool(entry.get("attach_only")) for entry in entries) + if mount and raw and raw != len(entries): + raise TypeError( + "mixed mounted and attach-only initial claims are not safe on older servers: " + "claim without volumes, then use set_volumes or patch_volumes" + ) + if raw: + mount = False + if not mount: + for entry in entries: + entry.pop("attach_only", None) if entries: claim["volumes"] = entries if not mount: diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index a892d173..a0a44a9b 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping from typing import Any, Final, Literal, TypedDict, cast VolumePhase = Literal["attached", "pending", "detaching"] @@ -54,14 +54,27 @@ def _volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | N or not isinstance(reply.get("converging"), bool) ): return None - return cast(SandboxVolumesResponse, reply) + view = dict(reply) + view["volumes_want"] = _response_volumes(reply["volumes_want"]) + view["volumes"] = _response_volumes(reply["volumes"]) + view["states"] = [dict(state) for state in reply["states"]] + return cast(SandboxVolumesResponse, view) + + +def _response_volumes(volumes: Iterable[Mapping[str, str | bool] | Volume]) -> list[Volume]: + # Response mounts identify raw devices; absent request mounts instead select a default path. + entries = [cast("Volume", dict(volume)) for volume in volumes] + for entry in entries: + if not entry.get("mount"): + entry["attach_only"] = True + return entries def _volume_bodies( - volumes: Sequence[str | Mapping[str, str | bool] | Volume], mount: bool + volumes: Iterable[str | Mapping[str, str | bool] | Volume], mount: bool ) -> list[dict[str, str | bool]]: - if isinstance(volumes, (str, bytes, bytearray)) or not isinstance(volumes, Sequence): - raise TypeError("volumes must be a sequence of volume names or mappings") + if isinstance(volumes, (str, bytes, bytearray, Mapping)) or not isinstance(volumes, Iterable): + raise TypeError("volumes must be an iterable of volume names or mappings") return [_volume_body(volume, mount) for volume in volumes] diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 88b220cf..73de4e74 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -119,7 +119,10 @@ def test_claim_attaches_volumes_without_mounting(node): "volumes_attach_only": True, } ] - assert sb.volumes == [{"name": "imagenet"}, {"name": "scratch", "mode": "rw"}] + assert sb.volumes == [ + {"name": "imagenet", "attach_only": True}, + {"name": "scratch", "mode": "rw", "attach_only": True}, + ] def test_template_claim_attaches_volumes_without_mounting(node): @@ -127,7 +130,7 @@ def test_template_claim_attaches_volumes_without_mounting(node): sb = Template(Client(node), node, "task:v1", "none", "small").new(volumes=["imagenet"], mount=False) assert seen[0]["volumes_attach_only"] is True assert seen[0]["volumes"] == [{"name": "imagenet"}] - assert sb.volumes == [{"name": "imagenet"}] + assert sb.volumes == [{"name": "imagenet", "attach_only": True}] def test_claim_rejects_mount_without_mounting(node): @@ -385,35 +388,33 @@ def kept_node(): @pytest.mark.parametrize("promoted", [False, True]) -def test_claim_preserves_mixed_volume_modes(node, promoted): - entries = [ - {"name": "mounted-ro"}, - {"name": "mounted-rw", "mount": "/work", "mode": "rw"}, - {"name": "device-ro", "attach_only": True}, - {"name": "device-rw", "mode": "rw", "attach_only": True}, - ] - seen = recording_claim({"id": "sb_1", "token": "tok", "volumes": entries}) +def test_claim_rejects_mixed_mounting_before_request(node, promoted): + entries = [{"name": "mounted-ro"}, {"name": "device-rw", "mode": "rw", "attach_only": True}] + seen = recording_claim({"id": "sb_1", "token": "tok"}) client = Client(node) - if promoted: - sb = Template(client, node, "task:v1", "none", "small").new(volumes=entries) - else: - sb = client.new("rt:24.04", volumes=entries) - assert seen[0]["volumes"] == entries - assert "volumes_attach_only" not in seen[0] - assert sb.volumes == entries + with pytest.raises(TypeError, match="claim without volumes"): + if promoted: + Template(client, node, "task:v1", "none", "small").new(volumes=entries) + else: + client.new("rt:24.04", volumes=entries) + assert seen == [] @pytest.mark.parametrize("promoted", [False, True]) -def test_claim_global_attach_only_keeps_per_entry_flags(node, promoted): - entries = [{"name": "data", "attach_only": False}, {"name": "work", "mode": "rw", "attach_only": True}] - seen = recording_claim({"id": "sb_1", "token": "tok"}) +@pytest.mark.parametrize("global_flag", [False, True]) +def test_claim_attach_only_uses_legacy_wire_encoding(node, promoted, global_flag): + entries = [{"name": "data", "attach_only": not global_flag}, {"name": "work", "mode": "rw", "attach_only": True}] + wire_entries = [{"name": "data"}, {"name": "work", "mode": "rw"}] + seen = recording_claim({"id": "sb_1", "token": "tok", "volumes": wire_entries}) client = Client(node) if promoted: - Template(client, node, "task:v1", "none", "small").new(volumes=entries, mount=False) + sb = Template(client, node, "task:v1", "none", "small").new(volumes=entries, mount=not global_flag) else: - client.new("rt:24.04", volumes=entries, mount=False) + sb = client.new("rt:24.04", volumes=entries, mount=not global_flag) assert seen[0]["volumes_attach_only"] is True - assert seen[0]["volumes"] == entries + assert seen[0]["volumes"] == wire_entries + assert sb.volumes == [{**entry, "attach_only": True} for entry in wire_entries] + assert entries[0]["attach_only"] is not global_flag @pytest.mark.parametrize("promoted", [False, True]) @@ -470,3 +471,53 @@ def test_claim_methods_preserve_empty_volume_sequence_semantics(node, promoted): claim(volumes=volumes) assert len(seen) == 3 assert all("volumes" not in body for body in seen) + + +@pytest.mark.parametrize("promoted", [False, True]) +@pytest.mark.parametrize("kind", ["list", "tuple", "generator", "set", "empty-generator"]) +def test_claim_volume_iterables_are_consumed_once(node, promoted, kind): + seen = recording_claim({"id": "sb_1", "token": "tok"}) + consumed = [] + + def generate(): + if kind != "empty-generator": + consumed.append("data") + yield "data" + + volumes = {"list": ["data"], "tuple": ("data",), "set": {"data"}}.get(kind) + if volumes is None: + volumes = generate() + client = Client(node) + if promoted: + Template(client, node, "task:v1", "none", "small").new(volumes=volumes) + assert seen[0].get("no_redirect", False) is (kind == "empty-generator") + else: + client.new("rt:24.04", volumes=volumes) + if kind == "empty-generator": + assert "volumes" not in seen[0] + else: + assert seen[0]["volumes"] == [{"name": "data"}] + assert consumed == (["data"] if kind == "generator" else []) + + +def test_claim_and_summary_response_inference_copies_wire_data(node): + client = Client(node) + reply = {"id": "sb_1", "token": "tok", "volumes": [{"name": "raw"}, {"name": "mounted", "mount": "/data"}]} + sb = client._handle_from(node, reply) + assert sb.volumes[0]["attach_only"] is True + assert "attach_only" not in sb.volumes[1] + assert "attach_only" not in reply["volumes"][0] + sb.volumes[0]["name"] = "caller" + assert sb.volumes_want[0]["name"] == reply["volumes"][0]["name"] == "raw" + rows = [ + {"id": "sb_1", "volumes": [{"name": "raw"}], "volumes_want": [{"name": "raw", "mount": ""}]}, + {"id": "sb_2", "volumes": [], "volumes_want": []}, + {"id": "sb_3"}, + ] + FakeNode.routes[("GET", "/v1/sandboxes")] = lambda body, path: (200, {"sandboxes": rows}) + summaries = client.sandboxes() + assert summaries[0]["volumes"][0]["attach_only"] is True + assert summaries[0]["volumes_want"][0]["attach_only"] is True + assert summaries[1]["volumes"] == summaries[1]["volumes_want"] == [] + assert "volumes" not in summaries[2] + assert "attach_only" not in rows[0]["volumes"][0] diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index c35724a7..59cc4a9e 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -46,8 +46,8 @@ def reply(body, path): @pytest.mark.parametrize("method", ["GET", "PUT", "PATCH"]) def test_volume_error_preserves_typed_partial_view(spawn_node, method): partial = { - "volumes_want": [{"name": "new", "mode": "rw", "attach_only": True}], - "volumes": [{"name": "old", "attach_only": True}], + "volumes_want": [{"name": "new", "mode": "rw"}], + "volumes": [{"name": "old"}], "converging": False, "states": [ {"name": "new", "state": "pending", "error": "retry"}, @@ -65,9 +65,14 @@ def test_volume_error_preserves_typed_partial_view(spawn_node, method): sb.patch_volumes(remove=["old"]) assert isinstance(caught.value, APIError) assert caught.value.status == 500 - assert caught.value.response == partial - assert sb.volumes_want == partial["volumes_want"] - assert sb.volumes == partial["volumes"] + expected = { + **partial, + "volumes_want": [{**partial["volumes_want"][0], "attach_only": True}], + "volumes": [{**partial["volumes"][0], "attach_only": True}], + } + assert caught.value.response == expected + assert sb.volumes_want == expected["volumes_want"] + assert sb.volumes == expected["volumes"] caught.value.response["volumes"][0]["name"] = "caller copy" assert sb.volumes[0]["name"] == "old" @@ -258,3 +263,32 @@ def reply(body, path): else: sb.patch_volumes(add=volumes) assert seen == [] + + +@pytest.mark.parametrize("method", ["set", "patch"]) +@pytest.mark.parametrize("kind", ["list", "tuple", "generator", "set", "empty-generator"]) +def test_live_volume_iterables_are_consumed_once(spawn_node, method, kind): + seen = [] + + def reply(body, path): + seen.append(body) + return 200, {"volumes_want": [], "volumes": [], "states": [], "converging": False} + + consumed = [] + + def generate(): + if kind != "empty-generator": + consumed.append("data") + yield "data" + + volumes = {"list": ["data"], "tuple": ("data",), "set": {"data"}}.get(kind) + if volumes is None: + volumes = generate() + sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) + if method == "set": + sb.set_volumes(volumes) + else: + sb.patch_volumes(add=volumes) + entries = [] if kind == "empty-generator" else [{"name": "data"}] + assert seen == [{"volumes" if method == "set" else "add": entries}] + assert consumed == (["data"] if kind == "generator" else []) From 70ce0e5f25f06cd0834185730909ad48163f413d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 03:19:09 +0000 Subject: [PATCH 22/72] docs: clarify live volume modes and compatibility --- README.md | 2 +- docs/deploy.md | 64 +++++++++++++----------- docs/live-volumes.md | 35 ++++++++++--- docs/sandboxd-api.md | 116 ++++++++++++++++++++++++++++--------------- docs/sdk-python.md | 73 ++++++++++++++++++++------- docs/sdk.md | 63 +++++++++++++++-------- docs/security.md | 19 ++++++- 7 files changed, 253 insertions(+), 119 deletions(-) diff --git a/README.md b/README.md index 04468e8d..68d48586 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ performance) — source in - `sandboxd/` — per-node control plane (Go): warm pools refilled from golden snapshot exports (online-retunable), claim/release/renew/hibernate/fork/ promote/checkpoint HTTP API, operator-catalog dataset volumes (read-only or - writable; read-only ones change on a running claim), signed preview URLs, + writable, mounted or attach-only; all support live changes), signed preview URLs, the HTTP-upgrade byte relays to silkd and to any guest port, read-only file access to hibernated or archived claims from a warm VM of their pool (opt-in per pool `reader`), claim env with diff --git a/docs/deploy.md b/docs/deploy.md index b72b59cc..cbe90b6d 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -253,7 +253,8 @@ previously only produced warnings now fail validation at startup. Remove or correct those entries before upgrading. Valid names for tenants that have not yet been created remain accepted. -Running claims can change mounted read-only catalog entries through the +Running claims can add or remove mounted and attach-only catalog entries in +read-only or writable mode through the [GET, PUT, and PATCH volume API](sandboxd-api.md#get-v1sandboxesidvolumes) and both SDKs. The image must already exist on the owner node. Live changes require Cocoon v0.7.2 or newer for idempotent disk attach and detach. PUT and @@ -266,8 +267,8 @@ The claim journal stores confirmed applied `Volumes`, plus desired `VolumesWant` while it differs. Admission holds cover their union across daemon restart and teardown. Retry an interrupted operation with the same PUT target. Release cancels convergence -and transfers holds to teardown. Capture is refused while either set is -nonempty. +and transfers holds to teardown. Capture is refused while either set, an active +operation, or unacknowledged recovery references still hold volumes. A sandbox whose VMM exits while holding volumes is release-only. Release it and create a new sandbox. Daemon restart recovery assumes the original VMM remains @@ -278,8 +279,9 @@ interrupted diagnostic after restart and require a caller retry. exercises GET, repeated full-set PUT, PATCH add and remove, and clearing a new claim. -Each catalog path must name a disk image containing a mountable whole-device -filesystem. Keep catalog names bound to the same image while any claim holds +Mounted entries need a disk image containing a mountable whole-device +filesystem. Attach-only entries leave device use and filesystem consistency +to the caller. Keep catalog names bound to the same image while any claim holds them. Use versioned names such as `dataset-v1` and `dataset-v2`; sandboxd cannot detect a repointed catalog entry. A read-only entry's image must stay immutable: do not replace, @@ -310,9 +312,11 @@ A volume claim may consume an ordinary warm Cloud Hypervisor VM. sandboxd attaches after the warm pop or provision, waits for the device settle — a serial match under `/sys/block`, then the `/dev/` node itself, since the kernel publishes sysfs before devtmpfs creates it — up to 2 seconds, then -mounts the device before finalizing the claim: `mode: "ro"` (the default) -attaches and mounts read-only; `mode: "rw"` requires the catalog entry's -`writable: true` and attaches and mounts read-write. Setup failure destroys +mounts each managed device before finalizing the claim. `mode: "ro"` (the default) +attaches read-only; `mode: "rw"` requires the catalog entry's +`writable: true` and attaches read-write. Per-entry `attach_only: true`, or +claim-wide `volumes_attach_only: true`, skips the device wait and mount. The +caller must wait for enumeration before using that device. Setup failure destroys the VM without quiescing — the claim was never handed out, so no workload write happened — and a popped warm VM is refilled normally. @@ -320,8 +324,8 @@ A multi-volume claim brings every volume up concurrently: each volume's own marker→attach→mount order is preserved, but cocoon serializes the hypervisor attach per VM, so what actually overlaps across volumes is the CLI spawns, device settles, and guest mounts. A partial failure still fails the whole -claim and destroys the VM without quiescing, so any `rw` volume already -attached keeps its dirty marker — including one the caller never got a handle +claim and destroys the VM without quiescing, so any mounted `rw` volume already +attached keeps its dirty marker, including one the caller never got a handle to; recovery is the normal `rw` cycle (see [Writable dataset volumes](#writable-dataset-volumes)). @@ -337,19 +341,20 @@ boot. the answering node's current local availability and whether the entry is `writable`, without exposing host paths or node addresses. Applied names, effective mounts, and (for `rw` entries) mode are persisted with the claim. -Such a claim cannot hibernate, fork, checkpoint, or promote; the idle -hibernate sweep leaves it running. Release removes the VM but never deletes -the operator-owned backing image — an `rw` release quiesces (unmounts) first, +While volume holds or unacknowledged recovery references remain, a claim cannot +hibernate, fork, checkpoint, or promote. The idle hibernate sweep leaves it +running. Release removes the VM but never deletes +the operator-owned backing image. A mounted `rw` release quiesces (unmounts) first, below. With `directio=off`, readers share the host page cache; use `directio=on` when cache interference matters. A dataset mounted into an egress-lane sandbox can be uploaded wherever that tenant's egress policy permits, so treat the ACL and egress policy as one access -decision. Image replication, detach, and refcounting remain out of scope. +decision. Image replication and business ownership remain external to sandboxd. #### Writable dataset volumes -A catalog entry with `writable: true` may be claimed with `mode: "rw"` -(omitted, or `"ro"`, always mounts read-only, even against a writable entry). +A catalog entry with `writable: true` may be claimed with `mode: "rw"`. +Omitted mode, or `"ro"`, attaches read-only, even against a writable entry. `directio` still defaults to `off` for a writable entry — the durability nuance is that the host page cache sits between the guest's flush and the device, so `directio=on` is the knob when that gap matters, not just cache @@ -362,7 +367,7 @@ so every claim for that name, `ro` or `rw`, already funnels to that one node; running the same writable name from two nodes at once is dataset divergence that nothing in the protocol detects for you. -**The `.dirty` marker.** Before the first `rw` attach of an image, +**The `.dirty` marker.** Before a mounted `rw` attach of an image, sandboxd durably creates a `.dirty` sidecar beside it; a clean `rw` release removes it. It is a write-ahead record, not a lock: it survives a killed sandboxd or a dead VMM, so anything short of a clean unmount leaves it @@ -375,6 +380,13 @@ can serve `ro` again. A `ro` claim against a dirty image is refused rather than auto-healed: replaying the journal takes the write lock, which conflicts with concurrent readers. +Live removal clears a mounted writer's marker only with current-call evidence +of the expected device at the managed root mount, writable mount and +superblock options, normal unmount, no remaining device mounts, and confirmed +host detach. Marker cleanup precedes release of the old hold. A prior external +unmount or absent device provides no such evidence. Detach can succeed while +the marker remains and read-only use still requires recovery. + **Concurrency, for one image name:** - `rw` ∥ `rw` — refused (409). @@ -386,17 +398,13 @@ with concurrent readers. proceed; a crashed writer leaves it, and `ro` claims are refused until one `rw` claim replays and releases cleanly. -**Attach-only claims opt out of the marker, not out of admission.** A claim -sent with `volumes_attach_only` gets the device attached and nothing else, so -sandboxd neither writes nor clears `.dirty` for it — it cannot verify a -mount it did not perform. Everything in this section still applies to the -default, mounting claims, and the exclusion rules above apply to attach-only -claims exactly the same way, which is what keeps other tenants safe. The -operator-visible difference: an image whose attach-only writer released -without unmounting cleanly carries no marker, so the next `ro` claim is -admitted and fails at mount time (500) instead of being refused early (409). -Whoever hands out attach-only `rw` access owns that trade; see -[sandboxd-api](sandboxd-api.md#attach-only-volumes). +**Attach-only entries have no daemon-managed dirty marker.** This applies to +both initial claims and live additions. sandboxd neither writes nor clears +`.dirty` for attach-only `rw`, including a guest crash during writes. +Reader and writer exclusion still applies, but a later `ro` claim can be +admitted without a marker and may see an inconsistent filesystem. Mount failure +is not guaranteed. The controller owns quiescence, unmount, crash recovery, +and safe read-only publication. See the [attach-only contract](sandboxd-api.md#attach-only-volumes). ### Reclaiming S3 storage diff --git a/docs/live-volumes.md b/docs/live-volumes.md index f787ecda..d71fef1f 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -104,17 +104,32 @@ restores the original processes. ## Writable and attach-only entries -Use `mode: "rw"` for a catalog entry that permits writing. Set per-entry `attach_only: true` when the caller mounts the device itself. Both choices can be -mixed with mounted read-only entries in PUT, PATCH, or a new claim: +Use `mode: "rw"` for a catalog entry that permits writing. Set per-entry +`attach_only: true` when the caller manages the device. PUT and PATCH accept +all four modes in a mixed list: ```json {"volumes":[{"name":"dataset"},{"name":"scratch","mode":"rw"},{"name":"raw-data","attach_only":true},{"name":"raw-work","mode":"rw","attach_only":true}]} ``` Omitting a mount still selects `/volumes/` for a new mounted entry. An -attach-only entry cannot supply a nonempty mount. Responses preserve the flag -through GET, PUT retries, and both SDKs. The existing Go claim option -`WithVolumesAttachOnly()` and Python `mount=False` remain claim-wide aliases. +attach-only entry cannot supply a nonempty mount. HTTP responses keep the +legacy `{name, mount?, mode?}` fields and omit `attach_only`. The current SDKs +infer the flag from a mountless response. Echoing that response to the same +sandbox retains the held device-only entry. An omitted or false flag does not +change that held shape. A mount or mode change requires a successful removal +before adding the name again. + +The existing Go claim option `WithVolumesAttachOnly()` and Python `mount=False` +remain claim-wide aliases. The current source SDKs also encode an all-attach-only +initial list with the legacy global flag. They reject mixed mounted and +attach-only initial claims before network I/O. Claim without volumes, as in the +example above, then use the live API for a mixed list. Direct HTTP mixed initial +claims require matching daemons across the serving fleet and redirect targets. +Older claim decoders can silently ignore per-entry flags. See +[version compatibility](sandboxd-api.md#volume-version-compatibility). These +per-entry SDK features are unreleased. Published Python 0.1.19 can still echo a +same-sandbox GET response but cannot construct the new mixed request mappings. Before removal, stop application access and close references. For attach-only devices, unmount the caller-managed filesystem first. sandboxd checks its guest @@ -123,10 +138,14 @@ mount namespaces are idle. Keep the control-plane reservation across any sleep and gate access until wake, attach, and mount complete. For mounted writable entries, sandboxd clears its dirty marker only after -observing a normal unmount of the expected writable root mount and confirming -host detach. External pre-unmount, an absent device, or a lost result leaves the +identifying the expected device at the managed root mount with writable mount +and superblock options, completing a normal unmount, confirming no residual +device mounts, and confirming host detach. The same removal must supply all +that evidence before the old hold is released. +External pre-unmount, an absent device, or a lost result leaves the marker conservatively in place. Detach can succeed while a subsequent read-only attach still requires recovery. Attach-only writers have no daemon-managed -marker and remain the caller's filesystem responsibility. See the [API +marker, including after a guest crash during writes. The caller owns recovery +and must establish consistency before read-only publication. See the [API contract](sandboxd-api.md#patch-v1sandboxesidvolumes) for failure and retry semantics. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index fd227fad..e46081ad 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -83,17 +83,17 @@ Auth: `Authorization: Bearer ` (when configured). - `egress` `false` claims with no [egress policy](egress.md), whatever the pool's (or, for a promoted template, its source pool's): the guest's doors are never bound. Omitted or `true` keeps the policy the claim would get -- `volumes` is an ordered list of at most eight unique catalog names. `mount` - defaults to `/volumes/`; a custom value must be absolute and clean, +- `volumes` is an ordered list of at most eight unique catalog names. Unless + `attach_only` is true, `mount` defaults to `/volumes/`; a custom value must be absolute and clean, outside the guest OS tree, unique, and non-nesting within the request. `mode` is `"ro"` (default, omitted) or `"rw"`; `"rw"` requires the catalog entry's `writable: true` (see [deploy](deploy.md#dataset-volumes)) - `volumes_attach_only` (default `false`) attaches every requested volume without mounting it, handing the whole mount contract to the workload. It - requires at least one volume, and rejects any entry carrying a `mount` — - the path is meaningless when the caller mounts the device. Everything below - describes the default, eager behaviour, which is completely unchanged; the - attach-only contract is in [its own section](#attach-only-volumes) + requires at least one volume and rejects entries with a nonempty `mount`. + Per-entry `attach_only: true` selects the same behavior for that entry. + See the [attach-only contract](#attach-only-volumes) and + [version compatibility](#volume-version-compatibility) before using mixed initial claims Success: @@ -129,23 +129,31 @@ and persisted at finalization — `mode` is omitted from the echo for `ro` entries, matching the request shape. sandboxd attaches each disk, waits for the device settle — a serial match under `/sys/block`, then the `/dev/` node itself, since the kernel publishes sysfs before devtmpfs creates it — up -to 2 seconds total, and mounts the filesystem — read-only, unless the entry -requested and was granted `rw` — before returning. A custom mount may shadow +to 2 seconds total, and mounts the filesystem for mounted entries before +returning. An attach-only entry skips that wait and mount. A custom mount may shadow an existing populated guest directory for the claim's life. ### Attach-only volumes Set `"attach_only": true` on an individual volume to stop after attachment. -It can share a claim with mounted entries. The existing claim-wide +It can share a raw HTTP claim with mounted entries on a matching daemon. +The existing claim-wide `"volumes_attach_only": true` applies that choice to every entry. A nonempty `mount` is invalid with either form. The device appears in the guest without a -mount. Echoed entries carry `attach_only: true` and no `mount` key: +mount. HTTP response entries omit both `mount` and `attach_only`: ```json {"id": "sb_…", "token": "…", - "volumes": [{"name": "imagenet", "attach_only": true}, {"name": "scratch-db", "mode": "rw", "attach_only": true}]} + "volumes": [{"name": "imagenet"}, {"name": "scratch-db", "mode": "rw"}]} ``` +Claim volume entries use this legacy shape in every HTTP response, including +nested sandbox summaries and partial failure views. The current SDKs infer attach-only from +a mountless response. Returning that response to the same sandbox preserves +the held attachment shape. Using it in a fresh claim requires an explicit +`attach_only: true` or the claim-wide flag. New entries without either flag +use the default mount. + Find each device by its virtio serial, which is the catalog name — poll this, not a one-shot lookup: the claim can return before the guest enumerates the device (typically within ~100ms, never guaranteed), and the `/dev/` node @@ -176,27 +184,26 @@ mounts it, so the guarantee does not depend on your mount flags. The whole consistency contract moves to the caller with the mount: -- sandboxd writes no dirty marker for an attach-only `rw` claim and clears - none, because it cannot verify your unmount and therefore promises nothing. - The flush is yours too: release runs no unmount and no `sync`, so whatever - the workload left unsynced is discarded with the VM. Releasing without a - clean unmount of your own leaves the image exactly as a filesystem crash - would. The next *eager* `ro` claim then fails at mount time with a 500 — - loud and attributable to the claim that skipped its unmount — instead of - the marker's 409. Recovery is any `rw` cycle that replays the journal. +- sandboxd writes and clears no dirty marker for attach-only `rw`. A guest + crash while writing also leaves no daemon-managed crash-dirty record. + Release runs no unmount and no `sync` for these entries. The caller must + stop writes, flush, and unmount before detach or release, and owns recovery + after a crash. A later `ro` claim can be admitted without a marker and may + mount an inconsistent filesystem. Mount failure is not guaranteed. Establish + filesystem consistency before publishing the image for read-only use. - a marker left by an earlier *eager* `rw` crash is not cleared by attach-only cycles, and still refuses eager `ro` claims with 409 until a `rw` claim releases cleanly. - writes straight to the block device bypass the filesystem journal entirely and are outside marker protection in either mode. -What still protects other claims is unchanged: admission excludes an -attach-only `rw` claim against every other claim of the name (and readers -against a writer), an attach-only `ro` claim of a marker-bearing image is -refused with 409, and a claim with volumes still refuses checkpoint, fork and -hibernate. An attach-only claim costs one disk attach per volume — no device -settle, no mount round-trip — and release costs nothing at all: removing the -VM closes the devices. +Admission excludes an attach-only `rw` claim against every other claim of the +name, and excludes a writer while readers hold the name. An attach-only `ro` +claim of a marker-bearing image is refused with 409. Capture remains refused +while volume holds or recovery references remain. An attach-only claim performs +one disk attach per volume without waiting for guest enumeration or mounting +it. Release removes the VM and +closes its devices without unmounting caller-managed filesystems. Redirects (mutually exclusive with the fields above) name peers to retry at — sent on a warm miss with warm peers, when the node lacks a golden for @@ -224,7 +231,7 @@ an unknown and a forbidden volume return the same error text. Errors: 400 unknown template axis, metadata over a bound (the message names it), an unknown `on_expire`, invalid/duplicate volumes, -`volumes_attach_only` with no volumes or with an entry carrying a `mount`, +`volumes_attach_only` with no volumes or with an entry carrying a nonempty `mount`, `mode: "rw"` against a non-writable entry, or a volume that is unknown or forbidden (the latter two are deliberately indistinguishable), or @@ -238,6 +245,23 @@ release before `ro` claims resume; see calling tenant at its own `max_claims`, or the node draining; 500 provisioning failed. +### Volume version compatibility + +Per-entry `attach_only` describes the current source API. Older claim decoders +can ignore that field and mount the disk instead. Before sending a mixed +initial claim over HTTP, upgrade every serving node, including redirect +targets. Do not retry a rejected live request after dropping `attach_only`. +An older live endpoint with strict request decoding rejects the new field +before changing devices. + +The current source Go and Python SDKs send all-attach-only initial claims using +the existing `volumes_attach_only` flag without per-entry flags. They reject +mixed mounted and attach-only initial claims locally, before network I/O. +Claim without volumes, then use PUT or PATCH for a mixed live list. These SDK +changes are unreleased. The published Python 0.1.19 SDK accepts only `name`, +`mount`, and `mode` in request mappings. HTTP responses retain those fields, +so its same-sandbox GET-to-PUT workflow remains supported. + ## GET /v1/volumes Auth: node API token (root or tenant). Lists the fleet catalog entries the @@ -360,7 +384,7 @@ and cross-node ownership. ## GET /v1/sandboxes/{id}/volumes -Returns the persisted desired set (`volumes_want`), confirmed applied set +Returns the current desired set (`volumes_want`), confirmed applied set (`volumes`), per-volume `states`, and `converging`. Auth requires a client API token. Root can access every sandbox; a tenant can access only its own. Sandbox tokens are rejected when authentication is enabled. The owning node @@ -406,11 +430,14 @@ Set `attach_only: true` on an entry to attach its device without mounting it. A nonempty `mount` with that flag is invalid. The legacy claim-wide `volumes_attach_only` flag still applies to every entry in a new claim. The claim-time name, mount, and eight-volume limits apply to the desired set. -A new entry with omitted `mount` defaults to `/volumes/`. Responses emit -`attach_only: true` for device-only entries, so GET's desired list can be sent -back as a PUT. Older clients can still retain a held mountless entry by -omitting both fields. To change an existing name's mount, mode, or attach-only -shape, first remove that name and wait for successful convergence. +A new entry without `attach_only: true` and with omitted `mount` defaults to +`/volumes/`. HTTP responses omit `attach_only` and omit `mount` for +device-only entries. GET's desired list can be echoed to the same sandbox. +For an existing held mountless entry, an empty or omitted mount with an omitted +or false `attach_only` preserves the attach-only shape. Explicit false is +equivalent to omission here. It does not request a change to a managed mount. +To change an existing name's nonempty mount, mode, or mounted shape, first +remove that name and wait for successful convergence. An identical target can be retried. Claims with `on_expire: archive` must change that policy before adding a volume. @@ -448,15 +475,19 @@ partitions, or other mount namespaces are idle. The controller owns those conditions. Mounted writable entries receive a dirty marker before attachment. Live removal -clears it only after sandboxd observes the expected writable root mount, -completes a normal unmount, confirms no remaining device mounts, and receives -successful host detach. Cleanup finishes before writer admission is released. +clears it only after sandboxd identifies the expected device at its managed root +mount and observes writable options for both that mount and its superblock. +The same removal must complete a normal unmount, confirm no remaining device +mounts, and receive successful host detach. Marker cleanup finishes before +writer admission is released. Missing, malformed, or read-only mount evidence +cannot prove a clean unmount. An already absent device, an externally unmounted filesystem, or uncertain host removal supplies no clean-unmount evidence. Detach can complete while the marker remains, and a later read-only request can return 409 requiring recovery. A marker-file removal failure is logged and retains the conservative marker. -Attach-only writers have no daemon-managed dirty marker; their caller owns -filesystem consistency. +Attach-only writers have no daemon-managed dirty marker, even after a guest +crash during writes. Their caller owns filesystem recovery and safe read-only +publication. ### Partial failure and recovery @@ -481,8 +512,13 @@ teardown; a teardown keeps them until the VM is confirmed gone. Failed or uncertain operations retain their holds until successful cleanup. Startup reserves those holds without replaying attachments. Release cancels convergence and transfers the held union to teardown without waiting for a blocked device call. Capture operations (hibernate, fork, checkpoint, and -promote) are refused while either set is nonempty or a convergence request -still owns admitted volume holds. +promote) are refused while either set is nonempty, a convergence request +still owns admitted volume holds, or unacknowledged recovery references remain. + +A successful removal response confirms device removal and retirement of the +old recovery reference. After an error or timeout, GET helps select the next +retry, but an empty view alone does not prove that retirement completed. +Retry the removal successfully before hibernating. Device calls honor cancellation and have a 30-second ceiling per volume; a caller that disconnects cancels the run, leaving interrupted states to retry. Journal writes diff --git a/docs/sdk-python.md b/docs/sdk-python.md index d726d366..6359bee6 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -106,16 +106,17 @@ sb = client.new("ghcr.io/cocoonstack/sandbox/rt:24.04", |---|---|---|---| | `net` | `"none"`, `"egress"` | `"none"` | Cloud Hypervisor network shape: `none` disables the NIC and uses vsock-only I/O; `egress` attaches a bridge/CNI NIC | | `size` | `"small"`, `"medium"`, `"large"`, `"xlarge"`, `"2xlarge"` | `"small"` | resource tier: 1cpu/512M, 2cpu/1G, 4cpu/4G, 4cpu/8G, 8cpu/16G | -| `volumes` | bare names or `{name, mount?, mode?}` mappings | `None` | attach and mount up to eight unique catalog dataset disks; an omitted mount defaults to `/volumes/`; `mode` is `"ro"` (default) or `"rw"` — `"rw"` requires the catalog entry's `writable: true`; accepted by `Client.new` and `Template.new` | +| `volumes` | iterable of bare names or `{name, mount?, mode?, attach_only?}` mappings | `None` | attach up to eight unique catalog dataset disks; mounted entries default to `/volumes/`; `mode` is `"ro"` (default) or `"rw"`, which requires the catalog entry's `writable: true`; `Client.new` and `Template.new` accept all-mounted or all-attach-only lists | | `mount` | bool | `True` | mount every requested volume. `False` attaches the devices and leaves the mounting to the workload; a mapping carrying `mount` is then a `TypeError` | | `ttl_seconds` | int | server default 5m | sandbox TTL, server-capped at 24h. The node reaps the sandbox after the TTL even if the client vanishes | | `deadline` | `time.monotonic()` value | `None` | keyword-only wall clock over the whole claim, redirect targets included: every request is bounded by whatever is left and a spent deadline raises `TimeoutError`. Without it each redirect candidate gets a fresh `timeout`, so a cluster with unreachable peers can cost a multiple of it. Also on `Template.new` and `Checkpoint.new` | `new` returns when the sandbox's silkd answers: a warm hit is milliseconds, a cold key can take the full boot. A volume claim may consume an ordinary warm -VM and returns only after every requested disk is mounted; `sb.volumes` contains -dictionaries with the finalized name, effective mount, and (for `rw` entries) -mode. Custom mounts must be absolute and clean, stay outside the guest OS +VM and returns after each mounted entry is ready. Attach-only entries skip +mounting and may still be awaiting guest device enumeration. `sb.volumes` +contains the finalized name, effective mount, mode, and attachment choice. +Custom mounts must be absolute and clean, stay outside the guest OS tree, and cannot duplicate or nest. The handle exposes `sb.id`, `sb.token`, `sb.owner`, `sb.deadline`, and `sb.from_checkpoint` (the lineage edge when @@ -123,20 +124,43 @@ branched). `sb.template_digest` is the exact content identity when the claim cloned a promoted template; it is empty for other sources. `Sandbox` is a context manager; `sb.close()` releases it (releasing one already gone is not an error — double-release and reap races stay silent). -Volume sandboxes cannot hibernate, fork, checkpoint, or promote. Checkpoint +A sandbox cannot hibernate, fork, checkpoint, or promote while volume holds or +unacknowledged recovery references remain. Checkpoint branches do not accept volumes of either mode in this version. -`mount=False` on `client.new` or `template.new` claims the same volumes -without mounting them: the dictionaries in `sb.volumes` carry no `mount` key, -and the workload finds each device by polling `/sys/block/*/serial` for the -catalog name — not guaranteed present when the claim returns, typically -within ~100ms — then confirms the `/dev/` node itself exists before -mounting. Everything above describes the default (`mount=True`) and is -unchanged by it. What changes is that the mount and its consistency are -entirely yours: sandboxd writes and clears no dirty marker for an attach-only -`rw` claim, because it cannot verify your unmount, so releasing without -unmounting cleanly leaves the image as a crash would — see -[sandboxd-api](sandboxd-api.md#attach-only-volumes) for the full contract. +`volumes` accepts lists, tuples, generators, and sets of names. The SDK consumes +an iterable once. A string, byte string, or single top-level mapping is not a +volume list. + +`mount=False` on `client.new` or `template.new` attaches every volume without +mounting it. Setting `attach_only: True` on every entry has the same effect. +Both forms send the legacy claim-wide flag and omit per-entry flags. Mixed +mounted and attach-only initial claims fail locally before an HTTP request. +Claim without volumes, then use the live API for a mixed list: + +```python +with client.new("rt:24.04") as sb: + view = sb.set_volumes([ + {"name": "dataset"}, + {"name": "scratch", "mode": "rw"}, + {"name": "raw-data", "attach_only": True}, + {"name": "raw-work", "mode": "rw", "attach_only": True}, + ]) +``` + +This avoids older claim decoders silently mounting an entry whose new flag +they ignore. See [version compatibility](sandboxd-api.md#volume-version-compatibility) +before sending mixed initial claims directly over HTTP. The per-entry SDK +features describe the current unreleased source. The published Python +0.1.19 SDK does not accept `attach_only` in request mappings. + +The current SDK exposes attach-only response entries with `attach_only: True` +and no `mount` key. Poll `/sys/block/*/serial` for the catalog name, then +confirm that the `/dev/` node exists before mounting. The claim may return +before either appears. sandboxd writes and clears no dirty marker for +attach-only `rw`, including a guest crash during writes. The caller owns +filesystem consistency, recovery, and safe read-only publication. See the +[attach-only contract](sandboxd-api.md#attach-only-volumes). The caller-visible constraints are deliberate: volume claims may consume a warm VM, remain non-capturable, mount read-only by default, and require Cloud @@ -161,18 +185,29 @@ peer advertising both the template and every volume; when that intersection is empty, one volume holder may self-verify access to a shared template store before provisioning. -## Changing read-only datasets +## Changing volumes Use `sb.get_volumes()`, `sb.set_volumes([...])`, and `sb.patch_volumes(add=[...], remove=[...])` on the same running sandbox. These methods use the client's API token on the owning node; a sandbox token cannot -manage datasets. `set_volumes([])` clears mutable entries; `None` is rejected. -PUT must retain writable and attach-only entries. PATCH retains omitted names. +manage volumes. Live changes support mounted and attach-only entries in either +mode, including mixed lists. `set_volumes([])` removes all entries; `None` is +rejected. PATCH retains omitted names. To change a held name's mount, mode, or +mounted shape, remove it successfully before adding it again. Each method takes `timeout=` (default: the client timeout, 120 s); a large replacement can need longer, and a call that times out cancels the server run, so retry the same target. Responses include `volumes_want`, `volumes`, `states`, and `converging`. +The HTTP entries keep the legacy `{name, mount?, mode?}` shape and omit +`attach_only`. The current SDK infers `attach_only: True` from mountless +response entries, including partial failure views. Published Python 0.1.19 +can still echo a response to the same sandbox with +`sb.set_volumes(sb.get_volumes()["volumes_want"])`. A new entry without a mount +uses the default mount. A held mountless entry retains attach-only semantics +when echoed with an omitted or false flag. Explicit nonempty mount or mode +changes still require removal first. + `VolumePhase` and `VOLUME_ATTACHED`, `VOLUME_PENDING`, and `VOLUME_DETACHING` name the three states. A pending state with `converging=False` needs an explicit retry. `VolumeError`, exported from `cocoonsandbox` and `cocoonsandbox.errors`, diff --git a/docs/sdk.md b/docs/sdk.md index 2d66f2aa..75e3175b 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -210,14 +210,15 @@ defer sb.Close() |---|---|---|---| | `WithNetwork(n)` | `NetNone`, `NetEgress` | `NetNone` | Cloud Hypervisor network shape: `NetNone` disables the NIC and uses vsock-only I/O; `NetEgress` attaches a bridge/CNI NIC | | `WithSize(s)` | `Small`, `Medium`, `Large`, `XLarge`, `XXLarge` | `Small` | resource tier: 1cpu/512M, 2cpu/1G, 4cpu/4G, 4cpu/8G, 8cpu/16G | -| `WithVolumes(volumes...)` | `Volume{Name, Mount?, Mode?}` entries | none | attach and mount up to eight unique catalog dataset disks; `Mount` defaults to `/volumes/`; `Mode` is `"ro"` (default) or `"rw"` — `"rw"` requires the catalog entry's `writable: true`; supported by `Client.New` and `Template.New` | +| `WithVolumes(volumes...)` | keyed `Volume` entries with `Name`, optional `Mount`, `Mode`, and `AttachOnly` | none | attach up to eight unique catalog dataset disks; mounted entries default to `/volumes/`; `Mode` is `"ro"` (default) or `"rw"`, which requires the catalog entry's `writable: true`; `Client.New` and `Template.New` accept all-mounted or all-attach-only lists | | `WithVolumesAttachOnly()` | — | mount | attach the requested volumes without mounting them; the workload finds each device and owns the mount. Rejects a `Volume.Mount` locally | | `WithTimeout(d)` | duration | server default 5m | sandbox TTL, rounded up to seconds, server-capped at 24h. The node reaps the sandbox after the TTL even if the client vanishes | `New` returns when the sandbox's silkd answers: a warm hit is milliseconds, a cold key can take the full boot. A volume claim may consume an ordinary warm -VM and returns only after every requested disk is mounted; the finalized -name, effective mount, and (for `rw` entries) mode are available in +VM and returns after each mounted entry is ready. Attach-only entries skip +mounting and may still be awaiting guest device enumeration. The finalized +name, effective mount, mode, and attachment choice are available in `Sandbox.Volumes`. Custom mounts must be absolute and clean, stay outside the guest OS tree, and cannot duplicate or nest. `Sandbox.ID`, `Sandbox.Deadline`, and @@ -227,15 +228,16 @@ promoted template; it is empty for other sources. `Owner()` names the owning node, and `Token()` returns the per-sandbox bearer to persist with `ID` for a later `Lookup`; `Close()` releases the sandbox (releasing one already gone is not an error, and `Close` is bounded internally so it stays defer-friendly). -Volume sandboxes cannot hibernate, fork, checkpoint, or promote. Passing +A sandbox cannot hibernate, fork, checkpoint, or promote while volume holds or +unacknowledged recovery references remain. Passing `WithVolumes` or `WithClaimRef` to `Checkpoint.New` returns a local error: checkpoint branches support neither volumes nor a claim reference in this version. `WithMetadata`, `WithArchiveOnExpire` and `WithEnv` apply to `New`, `Template.New`, and `Checkpoint.New`: the labels follow the [claim bounds](sandboxd-api.md#post-v1claim), and a branch never inherits its source's labels or expiry action. -For datasets chosen after the sandbox starts, set the desired read-only mounts -on the same claim: +For volumes chosen after the sandbox starts, set the desired set on the same +claim. Live changes support mounted and attach-only entries in either mode: ```go view, err := sb.SetVolumes(ctx, []sandbox.Volume{ @@ -243,7 +245,7 @@ view, err := sb.SetVolumes(ctx, []sandbox.Volume{ }) view, err = sb.GetVolumes(ctx) view, err = sb.PatchVolumes(ctx, sandbox.VolumePatch{Remove: []string{"imagenet"}}) -// An explicit empty slice clears mutable mounts. A nil slice is rejected. +// An explicit empty slice removes all entries. A nil slice is rejected. view, err = sb.SetVolumes(ctx, []sandbox.Volume{}) ``` @@ -265,6 +267,12 @@ can include an `error`. Methods use the client API token on the owning node and update both public sets from each complete response, including a partial failure. Serialize changes to one handle as with `Renew`. Returned views do not share the handle's slices or dictionaries. +HTTP responses keep the legacy `{name, mount?, mode?}` shape and omit +`attach_only`. The current SDK sets `Volume.AttachOnly` when decoding a +mountless response. An omitted mount in a new request still selects the +default mount. A mountless entry echoed to the same sandbox retains its +existing attach-only meaning, including when `AttachOnly` is false. + Go returns `*sandbox.VolumeError` with a `Response` alongside the returned view. It wraps `*sandbox.APIError`, so `errors.As` can inspect the status. Python raises `VolumeError`, a subclass of `APIError`, with `.response`. A response is absent @@ -274,29 +282,42 @@ the handle retains its last known sets. Retry an interrupted operation with the same full PUT target, or set a new target to recover. `Client.Sandboxes` exposes `VolumesWant` and `Volumes` for inspection with an API token. -A PUT must retain existing writable and attach-only entries unchanged. PATCH -cannot remove or change them. Paused sandboxes are not woken, and images must +A PUT replaces all entries, including writable and attach-only entries. PATCH +retains names it does not mention and can add or remove any of the four modes. +Changing a held name's mount, mode, or mounted shape requires a successful +remove before adding it again. Paused sandboxes are not woken, and images must exist on the owner node. Admission holds cover the union of desired and applied sets. A settled retained entry receives no new attach or mount calls. `converging` is true only during an active request. Pending or detaching entries with false require an explicit retry. Daemon restart replaces unfinished diagnostics with -an interrupted message. Capture is refused while either set is nonempty. +an interrupted message. Capture remains refused while either set or +unacknowledged recovery references hold volumes. Release cancels convergence and hands held volumes to teardown. See the [API recovery contract](sandboxd-api.md#partial-failure-and-recovery) and [same-sandbox replacement tutorial](live-volumes.md). A failed sandbox with volume holds is release-only after VMM exit; an empty target cannot recover it. -`WithVolumesAttachOnly()` claims the same volumes without mounting them: the -entries in `Sandbox.Volumes` carry an empty `Mount`, and the workload finds -each device by polling `/sys/block/*/serial` for the catalog name — not -guaranteed present when the claim returns, typically within ~100ms — then -confirms the `/dev/` node itself exists before mounting. Everything -above describes the default and is unchanged by this option. What changes is -that the mount and its consistency are entirely yours: sandboxd writes and -clears no dirty marker for an attach-only `rw` claim, because it cannot verify -your unmount, so releasing without unmounting cleanly leaves the image as a -crash would — see -[sandboxd-api](sandboxd-api.md#attach-only-volumes) for the full contract. +`WithVolumesAttachOnly()` claims the same volumes without mounting them. +Setting `AttachOnly: true` on every entry has the same effect. Both forms use +the legacy claim-wide flag on the wire and omit per-entry flags. Mixed +mounted and attach-only initial claims are rejected locally before any HTTP +request. Claim without volumes, then call `SetVolumes` or `PatchVolumes` with +the mixed list. This prevents older claim decoders from silently mounting an +entry whose new flag they ignore. See [version compatibility](sandboxd-api.md#volume-version-compatibility) +before sending a mixed initial claim directly over HTTP. + +Attach-only entries in `Sandbox.Volumes` have `AttachOnly: true` and an empty +`Mount`. Poll `/sys/block/*/serial` for the catalog name, then confirm that the +`/dev/` node exists before mounting. The claim may return before either +appears. sandboxd writes and clears no dirty marker for attach-only `rw`, even +if the guest crashes while writing. The caller owns filesystem consistency, +recovery, and safe read-only publication. See the [attach-only contract](sandboxd-api.md#attach-only-volumes). + +The added `AttachOnly` field breaks unkeyed Go struct literals. Replace +`sandbox.Volume{"data", "/data", "rw"}` with +`sandbox.Volume{Name: "data", Mount: "/data", Mode: "rw"}`. These per-entry +features describe the current unreleased source, not a new published SDK +version. The caller-visible constraints are deliberate: volume claims may consume a warm VM, remain non-capturable, mount read-only by default, and require Cloud diff --git a/docs/security.md b/docs/security.md index ee2787c0..97f6f611 100644 --- a/docs/security.md +++ b/docs/security.md @@ -143,8 +143,23 @@ Live writable and attach-only changes enforce the same catalog ACL and local reader and writer admission as initial claims. Attach-only callers own quiescence, unmount, and filesystem consistency. Guest mountinfo checks do not prove the absence of raw block users, partition users, or users in other mount namespaces. -A mounted writable dirty marker is cleared only with observed normal unmount -and confirmed host detach; missing evidence leaves it in place. +A mounted writable dirty marker is cleared only when the same removal observes +the expected device at the managed root mount, writable mount and superblock +options, successful normal unmount, no residual device mounts, and confirmed +host detach. Marker cleanup precedes release of the old hold. Missing evidence +leaves the marker in place. + +sandboxd writes and clears no daemon-managed marker for attach-only `rw`, even +when the guest crashes while writing. Local reader and writer exclusion does not prove +the image is consistent after detach or release. A later `ro` claim can be +admitted without a marker. The controller must recover and verify the image +before read-only publication. + +An external controller owns stop-write and unmount before detach, its own +reservation across sleep, and access gating until wake, attach, and mount +complete. Wake establishes silkd readiness, and business traffic may trigger +it. The controller must gate those entry points too. The API does not implement +or verify that composed lifecycle. See the [API contract](sandboxd-api.md#delete-v1volumesname). ## Known limitations From c560e6bc6f268a2cec6359993e8ae8aafbe25f07 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:07:24 +0000 Subject: [PATCH 23/72] pool: cover runtime and static backing path parity --- sandboxd/pool/volume_catalog_test.go | 37 ++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index 0c721752..e24430a2 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -141,20 +141,41 @@ func TestRuntimeCatalogInvalidDestinationDoesNotFenceStaticClaims(t *testing.T) } } -func TestRuntimeRegistrationRejectsNonregularBacking(t *testing.T) { +func TestRuntimeCatalogUsesStaticPathValidation(t *testing.T) { requireCatalogRoot(t) - for _, path := range []string{t.TempDir(), os.DevNull} { + regular := writeVolumeImage(t, "backing", "unchanged") + link := filepath.Join(t.TempDir(), "backing-link") + if err := os.Symlink(regular, link); err != nil { + t.Fatal(err) + } + for _, path := range []string{regular, link, os.DevNull, t.TempDir(), filepath.Join(t.TempDir(), "missing")} { t.Run(filepath.Base(path), func(t *testing.T) { + spec := config.VolumeSpec{Name: "runtime", Path: path, DirectIO: types.DirectIOOff, Writable: true} + if err := config.ValidateVolumeCatalog([]config.VolumeSpec{spec}); err != nil { + t.Fatalf("static validation: %v", err) + } m := newVolumeManager(t, newFakeEngine(), nil) - err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path, Writable: true}) - if !errors.Is(err, ErrBadVolume) { - t.Fatalf("nonregular backing registration: %v", err) + registration := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "runtime", registration); err != nil { + t.Fatalf("runtime validation differs from static: %v", err) + } + m2 := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil) + if err := m2.RegisterVolume(t.Context(), "runtime", registration); err != nil { + t.Fatalf("idempotent retry after restart: %v", err) } - if names := m.VolumeNames(); len(names) != 0 { - t.Fatalf("rejected backing was published: %v", names) + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + if _, resolveErr := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(resolveErr, ErrVolumeUnavailable) { + t.Fatalf("missing backing resolved: %v", resolveErr) + } + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) } }) } + if got, err := os.ReadFile(regular); err != nil || string(got) != "unchanged" { + t.Fatalf("registration touched backing: %q %v", got, err) + } } func TestRuntimeCatalogCollisionsAndRecoveryNames(t *testing.T) { @@ -182,7 +203,7 @@ func TestRuntimeCatalogCollisionsAndRecoveryNames(t *testing.T) { if err := m.UnregisterVolume(t.Context(), "unexplained"); !errors.Is(err, ErrVolumeRegistrationConflict) { t.Fatalf("absent referenced removal: %v", err) } - for _, spec := range []types.VolumeRegistration{{Path: "relative"}, {Path: other, DirectIO: "invalid"}, {Path: other + "missing"}} { + for _, spec := range []types.VolumeRegistration{{Path: "relative"}, {Path: other, DirectIO: "invalid"}} { if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrBadVolume) { t.Fatalf("invalid spec %+v: %v", spec, err) } From b67d0b3e6d6a85705bbc23832c77c6075f1a6142 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:08:23 +0000 Subject: [PATCH 24/72] pool: align runtime backing paths with static catalog rules --- docs/live-volumes.md | 2 +- docs/sandboxd-api.md | 10 +++++----- docs/security.md | 18 ++++++++++++------ sandboxd/pool/volume_catalog.go | 5 +---- sandboxd/pool/volume_catalog_test.go | 3 +-- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/docs/live-volumes.md b/docs/live-volumes.md index d71fef1f..e138ef31 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -9,7 +9,7 @@ file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. A sandbox token alone cannot manage volumes. -A configured node root token can [register existing regular backing files](sandboxd-api.md#put-v1volumesname) +A configured node root token can [register backing paths](sandboxd-api.md#put-v1volumesname) without editing static config. Registration persists only node-local catalog metadata and does not manage disk files. For a sleep and wake workflow, an external controller owns quiescence and unmount before detach, a reservation diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index e46081ad..00d4086c 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -288,7 +288,7 @@ Auth: the configured root node API token. With no configured `api_token`, registration and unregistration return 403. A tenant token returns 403. Sandbox, unknown, and missing tokens return 401 when `api_token` is configured. -Registers an existing backing file on this node. The URL supplies the name; +Registers a backing path on this node. The URL supplies the name; the body has no `name` field. Unknown fields, duplicate fields, and trailing JSON return 400. @@ -299,10 +299,10 @@ JSON return 400. - `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. - `path` uses the same absolute-path validation as static volume config. - Creating a registration requires an existing regular file. A symlink to a - regular file is allowed; directories and device files return 400. This check - runs only when creating the registration, not on idempotent retries or - attachment. Operators must keep the registered path and target stable. + Registration does not open the backing or check its type or availability. + A missing backing retains its registration but cannot be attached until available. + Registration success does not establish that Cocoon supports the backing type. + Operators must keep the registered path and target stable. - `directio` is `on`, `off`, or `auto`, and defaults to `off`. - `writable` defaults to `false` and controls writable claim admission. - `tenants` limits use to the named tenants. Omitted or empty permits all tenants. diff --git a/docs/security.md b/docs/security.md index 97f6f611..40ae2845 100644 --- a/docs/security.md +++ b/docs/security.md @@ -105,12 +105,18 @@ exposes host image paths or access lists. Runtime registration and unregistration require a configured root `api_token`; these endpoints return 403 when the token is unset. The daemon must also run -as OS root to change the root-owned registration file. Runtime registration -checks that the backing resolves to a regular file when creating a registration. -Symlinks to regular files are allowed. An idempotent registration retry does not -repeat this check, and attachment does not recheck the file type or pin the -backing identity. Operators must prevent backing paths or symlink targets from -being replaced while registered and coordinate aliases. +as OS root to change the root-owned registration file. Registration uses the +same path validation as static configuration. It does not verify backing type +or pin backing identity. Operators must supply a backing supported by Cocoon, +prevent path or symlink target replacement while registered, and coordinate aliases. + +Mounted writable volumes use a dirty sidecar at the configured path plus `.dirty`. +The sidecar directory must be writable and retain the marker across host restarts. +Paths directly under ephemeral directories such as `/dev` do not provide that guarantee. +An operator-controlled symlink in a durable directory keeps the sidecar beside +that symlink, but alternate paths to the same backing still need operator coordination. +Block-device backends are not qualified by registration. Attach-only writable +volumes have no dirty marker and require external consistency checks. Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index c73d8b35..1ac1359f 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -154,7 +154,7 @@ func (m *Manager) loadRuntimeVolumes() error { return nil } -// RegisterVolume durably binds a root-managed name to an existing backing file. +// RegisterVolume durably binds a root-managed name to a backing path. func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { if os.Geteuid() != 0 { return ErrVolumeCatalogReadOnly @@ -172,9 +172,6 @@ func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.Vo if err != nil || proposal == nil { return err } - if st, statErr := m.statVolume(normalized.Path); statErr != nil || !st.Mode().IsRegular() { - return fmt.Errorf("%w: backing for volume %q must be an available regular file", ErrBadVolume, name) - } return m.writeCatalog(ctx, name, proposal) } diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index e24430a2..f53dc873 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -531,8 +531,7 @@ func TestRuntimeCatalogCancellationBeforeAcceptanceDoesNotFenceClaims(t *testing requireCatalogRoot(t) m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - m.catalog.stat = func(path string) (os.FileInfo, error) { cancel(); return os.Stat(path) } + cancel() m.catalog.replace = func(string, []byte) error { t.Fatal("canceled proposal attempted a write"); return nil } err := m.RegisterVolume(ctx, "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}) if !errors.Is(err, context.Canceled) || errors.Is(err, ErrVolumeCatalogUnavailable) { From 187c5b141b0b88fbdba6d49fc4afd5a8fa44aacf Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:12:26 +0000 Subject: [PATCH 25/72] docs: explain runtime ACL defaults and inventory limits --- docs/live-volumes.md | 5 ++++- docs/sandboxd-api.md | 22 +++++++++++++++++++--- docs/security.md | 20 +++++++++++++------- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/docs/live-volumes.md b/docs/live-volumes.md index e138ef31..a32733b1 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -11,7 +11,10 @@ A sandbox token alone cannot manage volumes. A configured node root token can [register backing paths](sandboxd-api.md#put-v1volumesname) without editing static config. Registration persists only node-local catalog -metadata and does not manage disk files. For a sleep and wake workflow, an +metadata and does not manage disk files. An empty runtime `tenants` list allows +only root to initiate attachment, including attachment to a tenant-owned claim. +Keep `api_token` configured. List tenants explicitly to grant self-service; +that permission also allows reattachment to another claim after detach. For a sleep and wake workflow, an external controller owns quiescence and unmount before detach, a reservation through sleep, and access gating until wake, attach, and mount complete. That gate also covers business requests that wake the sandbox. Wake establishes diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 00d4086c..fa5c774b 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -305,7 +305,10 @@ JSON return 400. Operators must keep the registered path and target stable. - `directio` is `on`, `off`, or `auto`, and defaults to `off`. - `writable` defaults to `false` and controls writable claim admission. -- `tenants` limits use to the named tenants. Omitted or empty permits all tenants. +- `tenants` grants the named tenants self-service access. Omitted or empty + makes a runtime entry operator-only. Root can attach it to a tenant-owned + claim; that tenant cannot add it to another claim. Static config entries + retain their existing public default for an empty list. Names use the tenant API name syntax. A valid name may be registered before that tenant exists; it grants no access until the tenant is created. @@ -362,7 +365,11 @@ has retired. An external controller owns quiescence and guest unmount before detach. It confirms successful detach and recovery bookkeeping before hibernation, and -keeps its own volume reservation while the sandbox sleeps. The controller +keeps its own volume reservation while the sandbox sleeps. For controller-only attachment, leave the runtime entry's `tenants` empty and +keep `api_token` configured. A nonempty ACL allows those tenants to attach the +volume themselves after detach; it does not reserve the volume for one claim. +Removing `api_token` restores the daemon's unauthenticated root-scope mode, so +operator-only admission then provides no network authentication. The controller gates every access path, including business requests that wake the sandbox, until wake, attach, and mount complete. Wake establishes only silkd readiness. It does not restore volume attachments or application readiness. Controller @@ -371,11 +378,20 @@ verification is included here. Downgrading to a daemon without runtime registration requires draining runtime references or supplying equivalent static volume config. Pending live volume changes must finish before downgrade. +Earlier development builds treated empty runtime ACLs as public. Loading those +records now restricts new admission to root without discarding existing claims +or blocking their cleanup. Before downgrading to a build with the old ACL rule, +drain and unregister operator-only entries. Leaving their registrations in place +would make them public again; draining claims alone does not prevent that. + An unresolved stale VM keeps volume admission and catalog changes fenced while cleanup retries continue. Inspect success alone does not prove a complete disk inventory. A present VM with omitted disks, or an active creating owner, retains that fence until a complete running inventory or confirmed removal resolves it. -An empty inventory is indistinguishable from an omitted one and stays fenced. +Cocoon v0.7.3 omits the disk list when its inventory query fails, and does not +collect it for stopped VMs. An empty list is therefore not proof of absence. +This availability limit also affects static read-only admission. Allowing readers +would conflict with an unknown writer. Cleanup retries remain enabled. A warm VM name is not evidence that no volume is attached; attachment can succeed before claim finalization fails. Known inventories keep holds on their reported names. Volume-free claims continue to work. Path uniqueness uses diff --git a/docs/security.md b/docs/security.md index 40ae2845..12382fc8 100644 --- a/docs/security.md +++ b/docs/security.md @@ -94,9 +94,11 @@ rather than confirming existence, and operator surfaces answer tenants 403. The volume catalog is an operator-owned data boundary. A volume name and its access list must mean the same thing fleet-wide, although membership is -node-local. An empty entry `tenants` list permits every authenticated scope; a -nonempty list permits only those named tenants, while the root token always -has access. Access-list names must use valid tenant name syntax. A name that +node-local. An empty static entry `tenants` list permits every authenticated +scope. An empty runtime entry is operator-only. A nonempty list grants the +named tenants self-service access. Root can attach an operator-only runtime +entry to a tenant-owned claim. For other live additions, the claim owner must +satisfy the entry's ACL, even when root sends the request. Access-list names must use valid tenant name syntax. A name that does not yet identify a tenant grants no access until that tenant exists. Claim lookup returns byte-identical errors for an unknown and a forbidden name, and catalog discovery filters before replying, so a tenant cannot enumerate restricted entries by probing. Gossip carries names only. @@ -132,16 +134,20 @@ immutable and publish a new name/path for new content. Changing a running claim's volumes needs the root token or the owning tenant's API token. A sandbox token cannot, so a guest holding its own sandbox token cannot attach another dataset, and every addition passes the same access list -as a claim. +as a claim. An operator-only runtime addition instead checks the root caller. +Already-held references can finish repair or removal without a new ACL grant. +Keep `api_token` configured for controller-only admission; without it the daemon +treats network requests as root. An ACL does not reserve a detached disk across +sleep. The controller must retain that reservation and gate workload access. A writable entry (`writable: true`) adds a channel the read-only model does not have: whichever tenant is permitted to claim it `rw` changes what every other permitted tenant reads next, and any of them can be the writer in turn — a multi-tenant access list on a writable entry is a bidirectional channel between those tenants, not just a shared read. Recommend a writable -entry's `tenants` name exactly one tenant — an empty list permits every -authenticated scope, which for a writable entry means every tenant can write -to every other tenant's next read. A dataset that genuinely needs multiple +entry's `tenants` name exactly one tenant when tenant self-service is intended. +An empty static list permits every tenant to write to other tenants' next reads. +Use an empty runtime list when only the controller should initiate attachment. A dataset that genuinely needs multiple writers needs an out-of-band process for who writes when, which the catalog ACL does not provide. From 335c0ce58460cd008e2ec909a37bcc189b3c6ee3 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:09:47 +0000 Subject: [PATCH 26/72] pool: reproduce runtime catalog default access exposure --- sandboxd/pool/volume_acl_test.go | 116 +++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 sandboxd/pool/volume_acl_test.go diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go new file mode 100644 index 00000000..98cce329 --- /dev/null +++ b/sandboxd/pool/volume_acl_test.go @@ -0,0 +1,116 @@ +package pool + +import ( + "errors" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestRuntimeCatalogDefaultACL(t *testing.T) { + requireCatalogRoot(t) + for _, tenants := range [][]string{nil, {}} { + t.Run("empty", func(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m, _ := newTenantVolumeManager(t, eng, catalog) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: tenants} + if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { + t.Fatal(err) + } + for _, current := range []*Manager{m, restartACLManager(t, eng, m.dataDir, catalog)} { + if slices.ContainsFunc(current.Volumes("acme", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { + t.Error("runtime volume with no tenant grants is visible to a tenant") + } + if !slices.ContainsFunc(current.Volumes("", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { + t.Error("operator cannot see runtime volume") + } + if _, err := current.VolumePlacement(testKey, "acme", []string{"home"}); !errors.Is(err, ErrVolumeUnavailable) { + t.Errorf("tenant placement: %v", err) + } + if local, err := current.VolumePlacement(testKey, "", []string{"home"}); err != nil || !local { + t.Errorf("operator placement: %v %v", local, err) + } + if _, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme", Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW, AttachOnly: true}}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Errorf("tenant initial attachment: %v", err) + } + if _, err := current.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "data"}}); err != nil { + t.Errorf("public static volume changed: %v", err) + } + } + }) + } +} + +func TestRuntimeCatalogOperatorLiveAttachment(t *testing.T) { + requireCatalogRoot(t) + for _, patch := range []bool{false, true} { + t.Run(map[bool]string{false: "put", true: "patch"}[patch], func(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, nil) + if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true}); err != nil { + t.Fatal(err) + } + want := []types.Volume{{Name: "home", Mode: types.VolumeModeRW, AttachOnly: true}} + change := func(caller string, volumes []types.Volume) error { + if patch { + _, err := m.PatchVolumes(t.Context(), sb.ID, caller, types.VolumePatch{Add: volumes}) + return err + } + _, err := m.SetVolumes(t.Context(), sb.ID, caller, volumes) + return err + } + if err := change("acme", want); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("tenant fresh attachment: %v", err) + } + if err := change("", want); err != nil { + t.Fatalf("operator attachment to tenant claim: %v", err) + } + if err := change("acme", want); err != nil { + t.Fatalf("held retry: %v", err) + } + if _, err := m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Remove: []string{"home"}}); err != nil { + t.Fatalf("tenant cleanup: %v", err) + } + if err := change("acme", want); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("tenant fresh attachment after detach: %v", err) + } + if err := change("", want); err != nil { + t.Fatalf("operator reattach: %v", err) + } + m2 := restartACLManager(t, eng, m.dataDir, nil) + if err := m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if got, err := m2.GetVolumes(sb.ID, "acme"); err != nil || len(got.Volumes) != 1 { + t.Fatalf("operator attachment lost at restart: %+v %v", got, err) + } + if _, err := m2.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{}); err != nil { + t.Fatalf("recovered cleanup: %v", err) + } + }) + } +} + +func TestRuntimeCatalogOperatorKeepsExplicitOwnerACL(t *testing.T) { + requireCatalogRoot(t) + m, sb := newTenantVolumeManager(t, newFakeEngine(), nil) + if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Tenants: []string{"other"}}); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "home"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("operator bypassed explicit claim-owner ACL: %v", err) + } +} + +func restartACLManager(t *testing.T, eng *fakeEngine, dataDir string, catalog []config.VolumeSpec) *Manager { + t.Helper() + cfg := &config.Config{DataDir: dataDir, APIToken: "root", Volumes: catalog} + m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + return m +} From 13f4936cf1bf3c8b720e8ed822a9dd38ac3cc6fe Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:11:51 +0000 Subject: [PATCH 27/72] pool: keep ungranted runtime volumes operator-only --- sandboxd/pool/claim.go | 2 +- sandboxd/pool/volume.go | 25 ++++++++++++++++++++---- sandboxd/pool/volume_acl_test.go | 7 +++++++ sandboxd/pool/volume_cleanup_test.go | 2 +- sandboxd/pool/volume_convergence_test.go | 6 +++--- sandboxd/pool/volume_mutation.go | 18 ++++++++++------- sandboxd/pool/volume_mutation_test.go | 2 +- 7 files changed, 45 insertions(+), 17 deletions(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 3d3be48a..7cfc5b4b 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -388,7 +388,7 @@ func (m *Manager) admitClaim(tenant string, volumes []resolvedVolume) error { if err := m.quotaErr(1, tenant); err != nil { return err } - return m.reserveResolved(tenant, volumes, nil) + return m.reserveResolved(volumeAccess{tenant: tenant, operator: tenant == ""}, volumes, nil) } // quotaErr answers ErrQuota when the node takes no more claims; callers hold m.mu. diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index ad8e7482..4a624168 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -53,8 +53,21 @@ type catalogVolume struct { writable bool } +// volumeAccess separates the claim owner's ACL from the caller's operator authority. +type volumeAccess struct { + tenant string + operator bool +} + func (v catalogVolume) allowed(tenant string) bool { - return tenant == "" || len(v.tenants) == 0 || slices.Contains(v.tenants, tenant) + return v.allowedFor(volumeAccess{tenant: tenant, operator: tenant == ""}) +} + +func (v catalogVolume) allowedFor(access volumeAccess) bool { + if v.runtime && len(v.tenants) == 0 { + return access.operator + } + return access.tenant == "" || len(v.tenants) == 0 || slices.Contains(v.tenants, access.tenant) } // Volumes reports the caller-visible fleet catalog; an empty tenant means root. @@ -137,6 +150,10 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri } func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested []types.Volume) ([]resolvedVolume, error) { + return m.resolveVolumesFor(ctx, volumeAccess{tenant: tenant, operator: tenant == ""}, requested) +} + +func (m *Manager) resolveVolumesFor(ctx context.Context, access volumeAccess, requested []types.Volume) ([]resolvedVolume, error) { if len(requested) == 0 { return nil, nil } @@ -154,7 +171,7 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { entry, ok := catalog[volume.Name] - if !ok || !entry.allowed(tenant) { + if !ok || !entry.allowedFor(access) { return nil, ErrVolumeUnavailable } resolvedEntry, resolveErr := m.resolveCatalogEntry(ctx, volume, entry) @@ -454,7 +471,7 @@ func (m *Manager) catalogSnapshot() map[string]*catalogVolume { return maps.Clone(m.catalog.entries) } -func (m *Manager) reserveResolved(tenant string, volumes []resolvedVolume, owner *types.Sandbox) error { +func (m *Manager) reserveResolved(access volumeAccess, volumes []resolvedVolume, owner *types.Sandbox) error { var held []types.Volume if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { held = slices.Concat(heldVolumes(owner), m.store.recoveryVolumes(owner.ID)[owner.ID]) @@ -464,7 +481,7 @@ func (m *Manager) reserveResolved(tenant string, volumes []resolvedVolume, owner if entry == nil || entry.generation != v.generation { return ErrVolumeUnavailable } - if !slices.Contains(held, v.applied) && !entry.allowed(tenant) { + if !slices.Contains(held, v.applied) && !entry.allowedFor(access) { return ErrVolumeUnavailable } if v.applied.RW() && !entry.writable { diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go index 98cce329..c9f9de28 100644 --- a/sandboxd/pool/volume_acl_test.go +++ b/sandboxd/pool/volume_acl_test.go @@ -39,6 +39,13 @@ func TestRuntimeCatalogDefaultACL(t *testing.T) { if _, err := current.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "data"}}); err != nil { t.Errorf("public static volume changed: %v", err) } + rootClaim, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW, AttachOnly: true}}}) + if err != nil { + t.Fatalf("operator initial attachment: %v", err) + } + if err := current.Release(t.Context(), rootClaim.ID, Cred{Token: rootClaim.Token}); err != nil { + t.Fatalf("operator initial claim cleanup: %v", err) + } } }) } diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go index 58f1b015..a14101c5 100644 --- a/sandboxd/pool/volume_cleanup_test.go +++ b/sandboxd/pool/volume_cleanup_test.go @@ -455,7 +455,7 @@ func TestArchivedReleaseWithUnvalidatedVolumeRunHasNoCleanupHolds(t *testing.T) if sb.VMName != "" { t.Fatalf("archived VM name = %q", sb.VMName) } - ctx, run, err := m.beginVolumeRun(t.Context(), sb) + ctx, run, err := m.beginVolumeRun(t.Context(), sb, sb.Tenant) if err != nil { t.Fatal(err) } diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 659ec1a5..1a31c51a 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -81,7 +81,7 @@ func TestReaddedDetachIsPendingAtTheDurableIntentBoundary(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}); err == nil { t.Fatal("expected detach failure") } - ctx, run, err := m.beginVolumeRun(t.Context(), sb) + ctx, run, err := m.beginVolumeRun(t.Context(), sb, sb.Tenant) if err != nil { t.Fatal(err) } @@ -115,7 +115,7 @@ func TestReleaseBeforeReservationDoesNotLeakAdmission(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - ctx, run, err := m.beginVolumeRun(t.Context(), sb) + ctx, run, err := m.beginVolumeRun(t.Context(), sb, sb.Tenant) if err != nil { t.Fatal(err) } @@ -325,7 +325,7 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { case "desired": sb.VolumesWant = held case "admitted": - _, run, err := m.beginVolumeRun(t.Context(), sb) + _, run, err := m.beginVolumeRun(t.Context(), sb, sb.Tenant) if err != nil { t.Fatal(err) } diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index dfe090ae..cf709413 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -24,6 +24,7 @@ type volumeRun struct { cancel context.CancelFunc done chan struct{} holds []types.Volume + access volumeAccess } // GetVolumes reads a claim's desired and applied volumes; a tenant reaches only its own claims. @@ -75,7 +76,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v } var run *volumeRun if err = m.eng.CheckLiveVolumeSupport(ctx); err == nil { - ctx, run, err = m.beginVolumeRun(ctx, sb) + ctx, run, err = m.beginVolumeRun(ctx, sb, tenant) } if err != nil { view, _ = m.GetVolumes(id, tenant) @@ -107,7 +108,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v return view, err } } - resolved, err := m.resolveLiveAdditions(ctx, sb, additions, recovery) + resolved, err := m.resolveLiveAdditions(ctx, run.access, additions, recovery) if err != nil { return view, err } @@ -170,7 +171,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v return view, nil } -func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (context.Context, *volumeRun, error) { +func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox, caller string) (context.Context, *volumeRun, error) { for { if err := ctx.Err(); err != nil { return ctx, nil, err @@ -191,7 +192,10 @@ func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (contex continue } runCtx, cancel := context.WithCancel(ctx) - run := &volumeRun{cancel: cancel, done: make(chan struct{}), holds: heldVolumes(sb)} + run := &volumeRun{ + cancel: cancel, done: make(chan struct{}), holds: heldVolumes(sb), + access: volumeAccess{tenant: sb.Tenant, operator: caller == ""}, + } m.volumeRuns[sb] = run m.mu.Unlock() return runCtx, run, nil @@ -271,7 +275,7 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run sb.Transition.Unlock() return claimSnapshot{}, false, ErrVolumeCapture } - if err := m.reserveResolved(sb.Tenant, resolved, sb); err != nil { + if err := m.reserveResolved(run.access, resolved, sb); err != nil { m.mu.Unlock() sb.Transition.Unlock() return claimSnapshot{}, false, err @@ -493,7 +497,7 @@ func heldVolumes(sb *types.Sandbox) []types.Volume { return slices.Concat(sb.Volumes, volumeDifferenceByName(sb.VolumesWant, sb.Volumes)) } -func (m *Manager) resolveLiveAdditions(ctx context.Context, sb *types.Sandbox, additions, recovery []types.Volume) ([]resolvedVolume, error) { +func (m *Manager) resolveLiveAdditions(ctx context.Context, access volumeAccess, additions, recovery []types.Volume) ([]resolvedVolume, error) { resolved := make([]resolvedVolume, 0, len(additions)) for _, v := range additions { if slices.Contains(recovery, v) { @@ -504,7 +508,7 @@ func (m *Manager) resolveLiveAdditions(ctx context.Context, sb *types.Sandbox, a resolved = append(resolved, entry) continue } - entries, err := m.resolveVolumes(ctx, sb.Tenant, []types.Volume{v}) + entries, err := m.resolveVolumesFor(ctx, access, []types.Volume{v}) if err != nil { return nil, err } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index 910bca56..f39f22bf 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -850,7 +850,7 @@ func assertLiveJournal(t *testing.T, m *Manager, sb *types.Sandbox) { func beginAdmittedVolumeRun(t *testing.T, m *Manager, sb *types.Sandbox) *volumeRun { t.Helper() - _, run, err := m.beginVolumeRun(t.Context(), sb) + _, run, err := m.beginVolumeRun(t.Context(), sb, sb.Tenant) if err != nil { t.Fatal(err) } From 79ea9697fcb8f70cd618be80e6d0ccbd5190546d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:12:24 +0000 Subject: [PATCH 28/72] e2e: cover runtime volume controller admission --- e2e/volume_catalog_test.go | 132 +++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 e2e/volume_catalog_test.go diff --git a/e2e/volume_catalog_test.go b/e2e/volume_catalog_test.go new file mode 100644 index 00000000..0833feda --- /dev/null +++ b/e2e/volume_catalog_test.go @@ -0,0 +1,132 @@ +package e2e + +import ( + "encoding/json/v2" + "errors" + "net/http" + "os" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + sandbox "github.com/cocoonstack/sandbox/sdk/go" +) + +func TestRuntimeVolumeControllerAdmissionThroughHTTP(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("runtime catalog writes require OS root") + } + st := startTenantStack(t, "node-token", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, nil) + acme, err := sandbox.Connect(st.addr, sandbox.WithAPIToken("acme-token")) + if err != nil { + t.Fatal(err) + } + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "home.img", "private bytes"), "writable": true}) + if err != nil { + t.Fatal(err) + } + tenantScope := *st + tenantScope.token = "acme-token" + for _, scope := range []struct { + token string + want int + }{{"acme-token", http.StatusForbidden}, {"", http.StatusUnauthorized}, {"node-token", http.StatusNoContent}} { + caller := *st + caller.token = scope.token + if status, body := rawJSON(t, &caller, http.MethodPut, "/v1/volumes/home", string(request)); status != scope.want { + t.Fatalf("registration auth: status=%d want=%d body=%s", status, scope.want, body) + } + } + volumes, err := acme.Volumes(t.Context()) + if err != nil || len(volumes) != 0 { + t.Fatalf("operator-only catalog exposed to tenant: volumes=%v err=%v", volumes, err) + } + volume := sandbox.Volume{Name: "home", Mode: "rw", AttachOnly: true} + if unexpected, claimErr := acme.New(t.Context(), "rt:24.04", sandbox.WithVolumes(volume)); claimErr == nil { + _ = unexpected.Close() + t.Fatal("tenant claimed an operator-only volume") + } else if refusal, ok := errors.AsType[*sandbox.APIError](claimErr); !ok || refusal.Status != http.StatusBadRequest { + t.Fatalf("tenant claim refusal: %v", claimErr) + } + var forbidden []byte + for _, name := range []string{"home", "unknown"} { + status, body := rawJSON(t, &tenantScope, http.MethodPost, "/v1/claim", `{"template":"rt:24.04","volumes":[{"name":"`+name+`"}]}`) + if status != http.StatusBadRequest { + t.Fatalf("claim privacy for %s: status=%d body=%s", name, status, body) + } + if name == "home" { + forbidden = body + } else if string(body) != string(forbidden) { + t.Fatalf("forbidden and unknown responses differ: %s versus %s", forbidden, body) + } + } + first, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer second.Close() + if _, err = first.SetVolumes(t.Context(), []sandbox.Volume{volume}); err == nil { + t.Fatal("tenant PUT attached an operator-only volume") + } + if _, err = first.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err == nil { + t.Fatal("tenant PATCH attached an operator-only volume") + } + operator := st.client.Attach(st.addr, first.ID, "") + if view, attachErr := operator.SetVolumes(t.Context(), []sandbox.Volume{volume}); attachErr != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) { + t.Fatalf("root attachment to tenant claim: view=%+v err=%v", view, attachErr) + } + if view, detachErr := first.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}); detachErr != nil || len(view.Volumes) != 0 { + t.Fatalf("tenant cleanup of root attachment: view=%+v err=%v", view, detachErr) + } + if _, err = second.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err == nil { + t.Fatal("tenant reused the controller volume on another claim") + } +} + +func TestRuntimeVolumeExplicitACLAndStaticPublicThroughHTTP(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("runtime catalog writes require OS root") + } + st := startTenantStack(t, "node-token", []tenants.TenantSpec{ + {Name: "acme", Token: "acme-token"}, {Name: "beta", Token: "beta-token"}, + }, []config.VolumeSpec{{Name: "public", Path: writeVolumeImage(t, "public.img", "shared")}}) + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "private.img", "private"), "tenants": []string{"acme"}}) + if err != nil { + t.Fatal(err) + } + if status, body := rawJSON(t, st, http.MethodPut, "/v1/volumes/private", string(request)); status != http.StatusNoContent { + t.Fatalf("registration: %d %s", status, body) + } + for _, tc := range []struct { + token string + want []string + }{{"acme-token", []string{"private", "public"}}, {"beta-token", []string{"public"}}} { + client, connectErr := sandbox.Connect(st.addr, sandbox.WithAPIToken(tc.token)) + if connectErr != nil { + t.Fatal(connectErr) + } + volumes, listErr := client.Volumes(t.Context()) + var names []string + for _, volume := range volumes { + names = append(names, volume.Name) + } + if listErr != nil || !slices.Equal(names, tc.want) { + t.Fatalf("catalog ACL: names=%v want=%v err=%v", names, tc.want, listErr) + } + claim, claimErr := client.New(t.Context(), "rt:24.04", sandbox.WithVolumes(sandbox.Volume{Name: "public"})) + if claimErr != nil { + t.Fatal(claimErr) + } + defer claim.Close() + _, attachErr := claim.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{{Name: "private"}}}) + if (attachErr == nil) != (tc.token == "acme-token") { + t.Fatalf("explicit ACL admission for %s: %v", tc.token, attachErr) + } + } +} From 4fc77f46a6120580923221fbebaf3f17dae0a26f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:15:25 +0000 Subject: [PATCH 29/72] ci: exercise runtime volume HTTP authorization as root --- .github/workflows/sandboxd.yml | 12 ++++++++++++ sandboxd/pool/volume.go | 1 - 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sandboxd.yml b/.github/workflows/sandboxd.yml index 61c312b6..16321074 100644 --- a/.github/workflows/sandboxd.yml +++ b/.github/workflows/sandboxd.yml @@ -78,6 +78,18 @@ jobs: exit 1 fi + - name: Test e2e as root + working-directory: e2e + shell: bash + run: | + GOWORK=off go test -race -c -o "$RUNNER_TEMP/volume-e2e.test" . + sudo setpriv --bounding-set=-dac_override,-dac_read_search "$RUNNER_TEMP/volume-e2e.test" -test.count=1 -test.v -test.skip '^TestCaddyTLSCluster$' | tee "$RUNNER_TEMP/volume-e2e-test.log" + grep -q '^--- PASS: TestRuntimeVolumeControllerAdmissionThroughHTTP ' "$RUNNER_TEMP/volume-e2e-test.log" + grep -q '^--- PASS: TestRuntimeVolumeExplicitACLAndStaticPublicThroughHTTP ' "$RUNNER_TEMP/volume-e2e-test.log" + if grep -- '--- SKIP:' "$RUNNER_TEMP/volume-e2e-test.log"; then + exit 1 + fi + - name: Test the PostgreSQL meta store working-directory: sandboxd shell: bash diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 4a624168..82959558 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -53,7 +53,6 @@ type catalogVolume struct { writable bool } -// volumeAccess separates the claim owner's ACL from the caller's operator authority. type volumeAccess struct { tenant string operator bool From af9025af1987bba545ca438359e9f9efb493bb29 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:29:36 +0000 Subject: [PATCH 30/72] docs: compose volume detach and restore around hibernation --- docs/live-volumes.md | 3 +- docs/sandboxd-api.md | 15 ++-- docs/sdk-python.md | 3 + docs/volume-controller.md | 154 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 docs/volume-controller.md diff --git a/docs/live-volumes.md b/docs/live-volumes.md index a32733b1..af4044c8 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -7,7 +7,8 @@ Register both read-only catalog entries on the owning node and grant the sandbox's tenant access to them. Put `v1` and `v2` in each image's `/version` file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. -A sandbox token alone cannot manage volumes. +A sandbox token alone cannot manage volumes. For a complete sleep and wake sequence, +see [the external controller example](volume-controller.md). A configured node root token can [register backing paths](sandboxd-api.md#put-v1volumesname) without editing static config. Registration persists only node-local catalog diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index fa5c774b..e381f648 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -373,9 +373,9 @@ operator-only admission then provides no network authentication. The controller gates every access path, including business requests that wake the sandbox, until wake, attach, and mount complete. Wake establishes only silkd readiness. It does not restore volume attachments or application readiness. Controller -orchestration is external; no controller implementation or composed lifecycle -verification is included here. Downgrading to a daemon without runtime -registration requires draining runtime references or supplying equivalent +orchestration is external. The [controller example](volume-controller.md) +shows the sequence and the reservation and access-gate contracts it requires. +Downgrading to a daemon without runtime registration requires draining runtime references or supplying equivalent static volume config. Pending live volume changes must finish before downgrade. Earlier development builds treated empty runtime ACLs as public. Loading those @@ -420,7 +420,8 @@ when the volume is in both sets, `pending` when desired but not yet confirmed, or `detaching` when applied but no longer desired. An interrupted device call can leave the actual guest state different from the confirmed set. A failed PUT or PATCH adds a top-level `error` naming the refusal, and the affected -state may carry its own `error`. `converging` is true only while a request worker is running. +state may carry its own `error`. `converging` is true while a request worker +is running or a recovery-only reference awaits an acknowledged claim write. `pending` with `converging: false` requires an explicit retry; there is no background convergence. GET retains the last safe per-volume error for the current daemon lifetime. After daemon restart, unfinished states report an @@ -430,8 +431,10 @@ The sandbox index exposes both sets for operator inspection. ## PUT /v1/sandboxes/{id}/volumes Replaces the full desired set and converges synchronously. Auth matches GET. -The owner tenant's catalog access list applies to newly admitted names, including -for root. Retaining, repairing, or removing held names does not reauthorize +For static entries and runtime entries with a nonempty ACL, the owner tenant's +access list applies to newly admitted names, including for root. An empty runtime +ACL instead requires the root caller, which can attach it to a tenant-owned claim. +Retaining, repairing, or removing held names does not reauthorize them. A new image must exist on the owning node. The request must include a non-null `volumes` array. An explicit empty array removes all entries. diff --git a/docs/sdk-python.md b/docs/sdk-python.md index 6359bee6..87ecfe15 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -161,6 +161,9 @@ before either appears. sandboxd writes and clears no dirty marker for attach-only `rw`, including a guest crash during writes. The caller owns filesystem consistency, recovery, and safe read-only publication. See the [attach-only contract](sandboxd-api.md#attach-only-volumes). +For a root client managing a tenant claim, see the +[external volume controller example](volume-controller.md). It keeps the +controller reservation and access gate across detach, hibernate, and wake. The caller-visible constraints are deliberate: volume claims may consume a warm VM, remain non-capturable, mount read-only by default, and require Cloud diff --git a/docs/volume-controller.md b/docs/volume-controller.md new file mode 100644 index 00000000..e00b6a3a --- /dev/null +++ b/docs/volume-controller.md @@ -0,0 +1,154 @@ +# Hibernate a sandbox with an external volume controller + +Use the live-volume API to remove a data disk before hibernation and restore +it after wake. This example keeps the same claim and catalog registration. +The controller owns the disk reservation and workload access policy throughout. +It does not add disk lifecycle management to sandboxd. + +## Prepare the claim and controller + +Use a `net: none` claim on its owning node. Egress-lane claims cannot hibernate. +Keep its TTL valid across the sleep interval. Use the current Python SDK with +[per-entry attach-only support](sdk-python.md#changing-volumes), Cocoon v0.7.2 or newer, +and Cloud Hypervisor. + +Prepare and format the backing outside sandboxd. Register it on the claim's +node with the configured root token: + +```http +PUT /v1/volumes/home +Authorization: Bearer +Content-Type: application/json + +{"path":"/srv/volumes/home.raw","writable":true,"tenants":[]} +``` + +An empty runtime `tenants` list lets only root initiate attachment. A nonempty +list grants those tenants self-service attachment to their claims, including +reuse after detach. Keep `api_token` configured. The daemon's no-token mode +treats network callers as root. + +Bind a root SDK client to the existing tenant claim. Keep the sandbox token +for guest commands and hibernation; the root client token authorizes volume +changes. + +```python +from cocoonsandbox import Client + +root = Client(owner_addr, api_token=root_node_token) +operator = root.attach(owner_addr, sandbox_id, sandbox_token) +``` + +Supply these application-specific operations before running the functions below: + +- `assert_reserved` verifies that the controller still exclusively reserves this + backing for this claim. The reservation must survive controller restart and + the entire sleep interval. Keep it exclusively held during every step and + renew any lease as needed. An assertion alone is not a lock or lease renewal. + Serialize operations for the same reservation. +- `close_access` blocks and drains business requests before returning. It must + fail closed. Cover direct ports, guest API access, and requests that can wake + the claim. The controller's own maintenance calls may still reach silkd. +- `quiesce_and_unmount` stops and reaps writers, flushes application state and + filesystem writes, and ordinarily unmounts the volume. It raises if any step + fails. Do not use lazy or forced unmount to turn a busy result into success. +- `mount_and_verify` finds the block device by its catalog serial, mounts the + existing filesystem, and verifies its expected identity and data. It raises + before admitting business traffic if the disk, mount, or checks are wrong. + Start the application behind the closed gate if it needs restarting. +- `open_access` admits business traffic only after those checks have succeeded. + +These callbacks are not supplied by sandboxd. An in-process lock or a local +boolean is not a durable reservation or an access gate for other callers. +The controller must enforce their contracts even when a step fails or its +process restarts. Root-only attachment does not prevent a guest from accessing +an already attached disk, or prevent the tenant from removing it for cleanup. + +Use one attach-only writable volume for this example. The controller mounts it +explicitly and owns filesystem consistency after a crash. sandboxd writes no +`.dirty` marker for attach-only RW. Do not substitute an automatically mounted +RW entry without accounting for its [dirty-marker rules](sandboxd-api.md#attach-only-volumes). + +## Controller functions + +Copy this block into the controller and pass its operations as `VolumeHooks`. +The functions do not release the reservation. Before the final `open_access` +call, exceptions stop the sequence without attempting to open access or run +another lifecycle operation. If a gate hook fails or has an uncertain result, +that hook and the controller must recover the gate. These functions cannot +prove that an external gate closed or undo a partially completed reopen. + +```python +from collections.abc import Callable +from dataclasses import dataclass + +from cocoonsandbox import Sandbox + + +@dataclass(frozen=True) +class VolumeHooks: + assert_reserved: Callable[[], None] + close_access: Callable[[], None] + quiesce_and_unmount: Callable[[], None] + mount_and_verify: Callable[[], None] + open_access: Callable[[], None] + + +def hibernate_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + hooks.quiesce_and_unmount() + view = sb.patch_volumes(remove=[name]) + if view["volumes_want"] or view["volumes"] or view["states"] or view["converging"]: + raise RuntimeError("volume references remain; keep access closed") + sb.hibernate() + + +def resume_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + sb.exec("true") + volume = {"name": name, "mode": "rw", "attach_only": True} + view = sb.patch_volumes(add=[volume]) + if view["volumes_want"] != [volume] or view["volumes"] != [volume] or view["converging"]: + raise RuntimeError("volume attachment is not settled; keep access closed") + hooks.mount_and_verify() + hooks.open_access() +``` + +Call `resume_volume` for initial attachment to an empty running claim, or after +sleep. It wakes through a guest command, then attaches and mounts the disk. +Call `hibernate_volume` after the application has used the mounted disk. +Do not wrap the handle in a context manager that releases the claim when the +controller leaves the sleep function. + +The successful remove response is the completion acknowledgment. Its empty +view is an additional check, not a replacement for that acknowledgment. +The function removes only the named volume. If another volume remains, it +refuses to hibernate rather than silently detaching that disk too. + +## Recover without reopening access early + +If unmount is busy, leave the disk attached and the gate closed. Find and stop +the remaining users before retrying. A guest mount check cannot prove that no +raw-device users, partition users, or other mount namespaces exist. + +If a volume request fails, times out, or loses its response, keep the +reservation and gate. GET can show what remains, but an empty GET view alone +does not acknowledge durable retirement. Retry the same remove target and +require success before calling hibernate. Make the guest cleanup hook safe +for a retry where the filesystem is already unmounted. Do not blindly rerun +application writes, formatting, or an entire lifecycle sequence. + +If hibernate or wake fails, inspect claim state and resume from the confirmed +stage while retaining the reservation. `exec("true")` proves only silkd +readiness. The claim may already have woken because another request reached +it. It still has no data disk until attach, and no data mount until the guest +mount operation succeeds. Any attach, mount, or data-check failure leaves +business traffic blocked. + +After a successful cycle, verify the same claim ID, backing identity, disk +serial, filesystem identity, and expected payload. A new VMM process after +wake is expected. The backing is external to the memory snapshot. Keep the +reservation while the resumed claim uses it; release it only through the +controller's normal ownership process. From 0efdfb1f2e8b4757b59f075c2ac55f2c74b3c8a5 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:29:36 +0000 Subject: [PATCH 31/72] e2e: cover externally orchestrated volume hibernation --- e2e/volume_catalog_test.go | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/e2e/volume_catalog_test.go b/e2e/volume_catalog_test.go index 0833feda..70765d73 100644 --- a/e2e/volume_catalog_test.go +++ b/e2e/volume_catalog_test.go @@ -130,3 +130,93 @@ func TestRuntimeVolumeExplicitACLAndStaticPublicThroughHTTP(t *testing.T) { } } } + +func TestRuntimeVolumeDetachHibernateWakeThroughHTTP(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("runtime catalog writes require OS root") + } + st := startTenantStack(t, "node-token", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, nil) + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "home.img", "external bytes"), "writable": true}) + if err != nil { + t.Fatal(err) + } + if status, body := rawJSON(t, st, http.MethodPut, "/v1/volumes/home", string(request)); status != http.StatusNoContent { + t.Fatalf("registration: %d %s", status, body) + } + acme, err := sandbox.Connect(st.addr, sandbox.WithAPIToken("acme-token")) + if err != nil { + t.Fatal(err) + } + claim, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer claim.Close() + other, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer other.Close() + operator := st.client.Attach(st.addr, claim.ID, "") + otherOperator := st.client.Attach(st.addr, other.ID, "") + volume := sandbox.Volume{Name: "home", Mode: "rw", AttachOnly: true} + if _, err = operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err != nil { + t.Fatal(err) + } + if err = claim.Hibernate(t.Context()); err == nil { + t.Fatal("captured an attached disk") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusConflict { + t.Fatalf("attached capture refusal: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err == nil { + t.Fatal("root admitted a second writer before detach") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusConflict { + t.Fatalf("second writer refusal: %v", err) + } + view, err := operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || len(view.States) != 0 || view.Converging { + t.Fatalf("detach acknowledgment: %+v %v", view, err) + } + if err = claim.Hibernate(t.Context()); err != nil { + t.Fatal(err) + } + if !hibernated(t, st, claim.ID) { + t.Fatal("detach followed by hibernate did not sleep the claim") + } + registered, listErr := st.client.Volumes(t.Context()) + if listErr != nil || len(registered) != 1 || registered[0].Name != "home" || !registered[0].Writable { + t.Fatalf("registration lost across sleep: %+v %v", registered, listErr) + } + if _, err = other.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err == nil { + t.Fatal("tenant acquired operator-only disk during sleep") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusBadRequest { + t.Fatalf("sleeping reservation ACL refusal: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err != nil { + t.Fatalf("attachment hold incorrectly survived detach: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}); err != nil { + t.Fatal(err) + } + beforeWake := st.eng.volumeOpsLog() + if out, execErr := claim.Exec(t.Context(), "echo", "awake"); execErr != nil || out != "awake\n" { + t.Fatalf("implicit wake: %q %v", out, execErr) + } + if hibernated(t, st, claim.ID) { + t.Fatal("guest request did not wake the original claim") + } + if ops := st.eng.volumeOpsLog(); !slices.Equal(ops, beforeWake) { + t.Fatalf("wake unexpectedly operated on external disks: %v -> %v", beforeWake, ops) + } + view, err = claim.GetVolumes(t.Context()) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || len(view.States) != 0 || view.Converging { + t.Fatalf("wake unexpectedly restored volume references: %+v %v", view, err) + } + view, err = operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}) + if err != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) || view.Converging { + t.Fatalf("controller reattachment: %+v %v", view, err) + } + if status, body := rawJSON(t, st, http.MethodDelete, "/v1/volumes/home", ""); status != http.StatusConflict { + t.Fatalf("unregister attached restored disk: %d %s", status, body) + } +} From ad65a425cfeb027d78b875fa07638b8c25e16c8c Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:51:34 +0000 Subject: [PATCH 32/72] engine: reproduce empty mount source rejection --- sandboxd/engine/volume_live_test.go | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index fc1da9c4..7a7fecd7 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -74,6 +74,48 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { } } +func TestLiveVolumeAcceptsEmptyMountSource(t *testing.T) { + for _, tt := range []struct { + name, mount string + rw bool + }{ + {name: "mounted read-only", mount: "/data"}, + {name: "mounted writable", mount: "/data", rw: true}, + {name: "attach-only"}, + } { + t.Run(tt.name, func(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + const unrelated = "501 84 0:62 / /empty-source rw,relatime - tmpfs rw,size=1024k,inode64\n" + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + unrelated) + fake.execHook = func(argv []string) { + switch argv[0] { + case "mount": + mode := mountMode(tt.rw) + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + unrelated + "2 1 8:32 / /data " + mode + " - ext4 /dev/vdc " + mode + "\n") + case "umount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + unrelated) + } + } + fake.mu.Unlock() + e := New("cocoon", nil, nil, false, false, "") + if tt.mount != "" { + if err := e.MountLiveVolume(t.Context(), sock, spec.Name, tt.mount, tt.rw); err != nil { + t.Fatalf("mount with unrelated empty source: %v", err) + } + } + result, err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, tt.mount) + if err != nil || result.NormalUnmount != tt.rw { + t.Fatalf("detach with unrelated empty source: result=%+v err=%v", result, err) + } + }) + } +} + func TestLiveMountRefusesAnOccupiedPathBehindASymlink(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) From 577fd86224e30c8205c2a147272a90c024fd2723 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:51:11 +0000 Subject: [PATCH 33/72] engine: preserve empty mountinfo sources --- sandboxd/engine/volume_live.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index 06a24b05..e093e10b 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -129,15 +129,13 @@ func (e *Engine) volumeMounts(ctx context.Context, vsockSocket, device string) ( } var mounts []volumeMount for line := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") { - fields := strings.Fields(line) - if len(fields) < 10 { + mount, filesystem, ok := strings.Cut(line, " - ") + fields := strings.Fields(mount) + fsFields := strings.Split(filesystem, " ") + if !ok || len(fields) < 6 || len(fsFields) != 3 || fsFields[0] == "" || fsFields[2] == "" { return "", nil, errors.New("invalid guest mountinfo") } - separator := slices.Index(fields[6:], "-") + 6 - if separator < 6 || separator+4 != len(fields) { - return "", nil, errors.New("invalid guest mountinfo") - } - mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5], fields[separator+3]}) + mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5], fsFields[2]}) } return strings.TrimSpace(string(devNum)), mounts, nil } From 95c2cb0f799345aa0106850317186c06367a3dc6 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:49:26 +0000 Subject: [PATCH 34/72] pool: reproduce recovery-only attachment authorization --- sandboxd/pool/volume_acl_test.go | 82 +++++++++++++++++++++++++ sandboxd/pool/volume_retirement_test.go | 13 ++-- 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go index c9f9de28..08b59c61 100644 --- a/sandboxd/pool/volume_acl_test.go +++ b/sandboxd/pool/volume_acl_test.go @@ -2,6 +2,7 @@ package pool import ( "errors" + "os" "slices" "testing" @@ -112,6 +113,87 @@ func TestRuntimeCatalogOperatorKeepsExplicitOwnerACL(t *testing.T) { } } +func TestRuntimeCatalogRecoveryReferenceDoesNotAuthorizeAttach(t *testing.T) { + requireCatalogRoot(t) + for _, operation := range []string{"put", "patch", "reservation"} { + t.Run(operation, func(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, nil) + if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true}); err != nil { + t.Fatal(err) + } + want := []types.Volume{{Name: "home", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", want); err != nil { + t.Fatalf("still-held retry: %v", err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), sb.ID, "acme", nil) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detached result: %+v, %v", view, err) + } + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil || claims[sb.ID] == nil || !slices.Equal(claims[sb.ID].Volumes, want) { + t.Fatalf("old durable attachment missing: %+v, %v", claims, err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: want}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("recovery reference lost writer exclusion: %v", err) + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture ignored recovery reference: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "home"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister ignored recovery reference: %v", err) + } + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + before := len(eng.volumeSpecs) + switch operation { + case "put": + _, err = m.SetVolumes(t.Context(), sb.ID, "acme", want) + case "patch": + _, err = m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Add: want}) + case "reservation": + resolved, resolveErr := m.resolveVolumes(t.Context(), "", want) + if resolveErr != nil { + t.Fatal(resolveErr) + } + _, run, beginErr := m.beginVolumeRun(t.Context(), sb, "acme") + if beginErr != nil { + t.Fatal(beginErr) + } + m.mu.Lock() + err = m.reserveResolved(run.access, resolved, sb) + m.mu.Unlock() + m.finishVolumeRun(sb, run) + } + if !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("recovery-only reference authorized tenant attachment: %v", err) + } + if len(eng.volumeSpecs) != before { + t.Fatal("rejected tenant reached device attachment") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatalf("operator reattach: %v", err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", nil); err != nil { + t.Fatalf("tenant cleanup after operator reattach: %v", err) + } + }) + } +} + func restartACLManager(t *testing.T, eng *fakeEngine, dataDir string, catalog []config.VolumeSpec) *Manager { t.Helper() cfg := &config.Config{DataDir: dataDir, APIToken: "root", Volumes: catalog} diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go index f4187b58..8b80fec9 100644 --- a/sandboxd/pool/volume_retirement_test.go +++ b/sandboxd/pool/volume_retirement_test.go @@ -236,7 +236,7 @@ func TestClaimRecoveryAttemptsRetainOnlyDistinctReferences(t *testing.T) { } } -func TestRecoveryOnlyRetryPreservesHeldRepairAuthorization(t *testing.T) { +func TestRecoveryOnlyRetryRequiresNewAuthorization(t *testing.T) { eng := newFakeEngine() m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}}); err != nil { @@ -265,11 +265,14 @@ func TestRecoveryOnlyRetryPreservesHeldRepairAuthorization(t *testing.T) { if err := os.Remove(tmp); err != nil { t.Fatal(err) } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { - t.Fatalf("exact recovery retry lost held authorization: %v", err) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); !errors.Is(err, ErrTenantRemoved) { + t.Fatalf("recovery-only retry bypassed tenant removal: %v", err) } - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { - t.Fatalf("repair reservation count: %+v", got) + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("rejected recovery retry reserved a reader: %+v", got) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("recovery retirement after tenant removal: %v", err) } if _, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "other"}}}); !errors.Is(err, ErrTenantRemoved) { t.Fatalf("repair exemption admitted unrelated addition: %v", err) From dec18ec8c59680d11820c02f4554aa252cccd3ab Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:50:01 +0000 Subject: [PATCH 35/72] pool: authorize recovery-only volume additions --- sandboxd/pool/volume.go | 2 +- sandboxd/pool/volume_mutation.go | 24 ++---------------------- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 82959558..9f867147 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -473,7 +473,7 @@ func (m *Manager) catalogSnapshot() map[string]*catalogVolume { func (m *Manager) reserveResolved(access volumeAccess, volumes []resolvedVolume, owner *types.Sandbox) error { var held []types.Volume if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { - held = slices.Concat(heldVolumes(owner), m.store.recoveryVolumes(owner.ID)[owner.ID]) + held = heldVolumes(owner) } for _, v := range volumes { entry := m.catalog.entries[v.applied.Name] diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index cf709413..eba3eeea 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -103,12 +103,12 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v return view, err } additions := volumeDifferenceByName(target, held) - if slices.ContainsFunc(additions, func(v types.Volume) bool { return !slices.Contains(recovery, v) }) { + if len(additions) > 0 { if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { return view, err } } - resolved, err := m.resolveLiveAdditions(ctx, run.access, additions, recovery) + resolved, err := m.resolveVolumesFor(ctx, run.access, additions) if err != nil { return view, err } @@ -496,23 +496,3 @@ func validateDetachNames(names []string) error { func heldVolumes(sb *types.Sandbox) []types.Volume { return slices.Concat(sb.Volumes, volumeDifferenceByName(sb.VolumesWant, sb.Volumes)) } - -func (m *Manager) resolveLiveAdditions(ctx context.Context, access volumeAccess, additions, recovery []types.Volume) ([]resolvedVolume, error) { - resolved := make([]resolvedVolume, 0, len(additions)) - for _, v := range additions { - if slices.Contains(recovery, v) { - entry, err := m.resolveCatalogVolume(ctx, v) - if err != nil { - return nil, err - } - resolved = append(resolved, entry) - continue - } - entries, err := m.resolveVolumesFor(ctx, access, []types.Volume{v}) - if err != nil { - return nil, err - } - resolved = append(resolved, entries...) - } - return resolved, nil -} From e705541e6c5a71d94a250db486e34d42b3806d81 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:51:47 +0000 Subject: [PATCH 36/72] pool: make catalog access policies explicit --- sandboxd/pool/claim.go | 2 +- sandboxd/pool/pool.go | 2 +- sandboxd/pool/volume.go | 48 +++++++++++++++++++----- sandboxd/pool/volume_acl_test.go | 12 +++--- sandboxd/pool/volume_catalog.go | 18 ++++++--- sandboxd/pool/volume_convergence_test.go | 2 +- sandboxd/pool/volume_mutation.go | 2 +- sandboxd/pool/volume_retirement_test.go | 2 +- 8 files changed, 62 insertions(+), 26 deletions(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 7cfc5b4b..6b20f440 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -388,7 +388,7 @@ func (m *Manager) admitClaim(tenant string, volumes []resolvedVolume) error { if err := m.quotaErr(1, tenant); err != nil { return err } - return m.reserveResolved(volumeAccess{tenant: tenant, operator: tenant == ""}, volumes, nil) + return m.reserveResolved(volumeAccessFor(tenant, tenant), volumes, nil) } // quotaErr answers ErrQuota when the node takes no more claims; callers hold m.mu. diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 729cd6c9..ba77428c 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -486,7 +486,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg m.catalog.entries[volume.Name] = &catalogVolume{ generation: m.catalog.generation, disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: cmp.Or(volume.DirectIO, types.DirectIOOff)}, - tenants: slices.Clone(volume.Tenants), + acl: newVolumeACL(volume.Tenants, volumePublic), writable: volume.Writable, } } diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 9f867147..407ce314 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -49,24 +49,54 @@ type catalogVolume struct { generation uint64 runtime bool disk engine.VolumeSpec - tenants []string + acl volumeACL writable bool } +type volumeACLMode uint8 + +const ( + volumeOperatorOnly volumeACLMode = iota + volumePublic + volumeTenantList +) + +type volumeACL struct { + mode volumeACLMode + tenants []string +} + type volumeAccess struct { - tenant string + owner string operator bool } +func newVolumeACL(tenants []string, fallback volumeACLMode) volumeACL { + if len(tenants) > 0 { + return volumeACL{mode: volumeTenantList, tenants: slices.Clone(tenants)} + } + return volumeACL{mode: fallback} +} + +func volumeAccessFor(caller, owner string) volumeAccess { + return volumeAccess{owner: owner, operator: caller == ""} +} + func (v catalogVolume) allowed(tenant string) bool { - return v.allowedFor(volumeAccess{tenant: tenant, operator: tenant == ""}) + return v.acl.allowedFor(volumeAccessFor(tenant, tenant)) } -func (v catalogVolume) allowedFor(access volumeAccess) bool { - if v.runtime && len(v.tenants) == 0 { +func (acl volumeACL) allowedFor(access volumeAccess) bool { + switch acl.mode { + case volumePublic: + return true + case volumeTenantList: + return access.owner == "" || slices.Contains(acl.tenants, access.owner) + case volumeOperatorOnly: return access.operator + default: + return false } - return access.tenant == "" || len(v.tenants) == 0 || slices.Contains(v.tenants, access.tenant) } // Volumes reports the caller-visible fleet catalog; an empty tenant means root. @@ -149,7 +179,7 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri } func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested []types.Volume) ([]resolvedVolume, error) { - return m.resolveVolumesFor(ctx, volumeAccess{tenant: tenant, operator: tenant == ""}, requested) + return m.resolveVolumesFor(ctx, volumeAccessFor(tenant, tenant), requested) } func (m *Manager) resolveVolumesFor(ctx context.Context, access volumeAccess, requested []types.Volume) ([]resolvedVolume, error) { @@ -170,7 +200,7 @@ func (m *Manager) resolveVolumesFor(ctx context.Context, access volumeAccess, re resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { entry, ok := catalog[volume.Name] - if !ok || !entry.allowedFor(access) { + if !ok || !entry.acl.allowedFor(access) { return nil, ErrVolumeUnavailable } resolvedEntry, resolveErr := m.resolveCatalogEntry(ctx, volume, entry) @@ -480,7 +510,7 @@ func (m *Manager) reserveResolved(access volumeAccess, volumes []resolvedVolume, if entry == nil || entry.generation != v.generation { return ErrVolumeUnavailable } - if !slices.Contains(held, v.applied) && !entry.allowedFor(access) { + if !slices.Contains(held, v.applied) && !entry.acl.allowedFor(access) { return ErrVolumeUnavailable } if v.applied.RW() && !entry.writable { diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go index 08b59c61..d08b9cbc 100644 --- a/sandboxd/pool/volume_acl_test.go +++ b/sandboxd/pool/volume_acl_test.go @@ -146,16 +146,16 @@ func TestRuntimeCatalogRecoveryReferenceDoesNotAuthorizeAttach(t *testing.T) { if err != nil || claims[sb.ID] == nil || !slices.Equal(claims[sb.ID].Volumes, want) { t.Fatalf("old durable attachment missing: %+v, %v", claims, err) } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: want}); !errors.Is(err, ErrVolumeBusy) { + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: want}); !errors.Is(err, ErrVolumeBusy) { t.Fatalf("recovery reference lost writer exclusion: %v", err) } - if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { t.Fatalf("capture ignored recovery reference: %v", err) } - if err := m.UnregisterVolume(t.Context(), "home"); !errors.Is(err, ErrVolumeRegistrationConflict) { + if err = m.UnregisterVolume(t.Context(), "home"); !errors.Is(err, ErrVolumeRegistrationConflict) { t.Fatalf("unregister ignored recovery reference: %v", err) } - if err := os.Remove(tmp); err != nil { + if err = os.Remove(tmp); err != nil { t.Fatal(err) } before := len(eng.volumeSpecs) @@ -184,10 +184,10 @@ func TestRuntimeCatalogRecoveryReferenceDoesNotAuthorizeAttach(t *testing.T) { if len(eng.volumeSpecs) != before { t.Fatal("rejected tenant reached device attachment") } - if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + if _, err = m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { t.Fatalf("operator reattach: %v", err) } - if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", nil); err != nil { + if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", nil); err != nil { t.Fatalf("tenant cleanup after operator reattach: %v", err) } }) diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go index 1ac1359f..375d5b89 100644 --- a/sandboxd/pool/volume_catalog.go +++ b/sandboxd/pool/volume_catalog.go @@ -55,7 +55,7 @@ func (c *volumeCatalog) admissionError() error { } func (v catalogVolume) registration() types.VolumeRegistration { - return types.VolumeRegistration{Path: v.disk.Path, DirectIO: v.disk.DirectIO, Writable: v.writable, Tenants: v.tenants} + return types.VolumeRegistration{Path: v.disk.Path, DirectIO: v.disk.DirectIO, Writable: v.writable, Tenants: v.acl.tenants} } func catalogDefinitions(entries map[string]*catalogVolume) []config.VolumeSpec { @@ -63,7 +63,7 @@ func catalogDefinitions(entries map[string]*catalogVolume) []config.VolumeSpec { for name, entry := range entries { volumes = append(volumes, config.VolumeSpec{ Name: name, Path: entry.disk.Path, DirectIO: entry.disk.DirectIO, - Writable: entry.writable, Tenants: entry.tenants, + Writable: entry.writable, Tenants: entry.acl.tenants, }) } return volumes @@ -75,8 +75,11 @@ func (c *volumeCatalog) adding(name string, spec types.VolumeRegistration) (*cat } entries := maps.Clone(c.entries) entries[name] = &catalogVolume{ - disk: engine.VolumeSpec{Name: name, Path: spec.Path, DirectIO: spec.DirectIO}, - tenants: spec.Tenants, writable: spec.Writable, runtime: true, generation: c.generation + 1, + generation: c.generation + 1, + runtime: true, + disk: engine.VolumeSpec{Name: name, Path: spec.Path, DirectIO: spec.DirectIO}, + acl: newVolumeACL(spec.Tenants, volumeOperatorOnly), + writable: spec.Writable, } if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { return nil, fmt.Errorf("%w: %w", ErrVolumeRegistrationConflict, err) @@ -143,8 +146,11 @@ func (m *Manager) loadRuntimeVolumes() error { } generation++ entries[name] = &catalogVolume{ - disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, - tenants: normalized.Tenants, writable: normalized.Writable, runtime: true, generation: generation, + generation: generation, + runtime: true, + disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, + acl: newVolumeACL(normalized.Tenants, volumeOperatorOnly), + writable: normalized.Writable, } } if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 1a31c51a..c112c859 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -58,7 +58,7 @@ func TestRemovedTenantRetainsAndRepairsHeldVolumes(t *testing.T) { } eng.detachVolumeErr = nil entry := m.catalog.entries["data"] - entry.tenants = []string{"other-tenant"} + entry.acl = newVolumeACL([]string{"other-tenant"}, volumePublic) m.catalog.entries["data"] = entry if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { t.Fatalf("held repair checked new authorization: %v", err) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index eba3eeea..03e6a7d2 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -194,7 +194,7 @@ func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox, caller runCtx, cancel := context.WithCancel(ctx) run := &volumeRun{ cancel: cancel, done: make(chan struct{}), holds: heldVolumes(sb), - access: volumeAccess{tenant: sb.Tenant, operator: caller == ""}, + access: volumeAccessFor(caller, sb.Tenant), } m.volumeRuns[sb] = run m.mu.Unlock() diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go index 8b80fec9..64f66e57 100644 --- a/sandboxd/pool/volume_retirement_test.go +++ b/sandboxd/pool/volume_retirement_test.go @@ -259,7 +259,7 @@ func TestRecoveryOnlyRetryRequiresNewAuthorization(t *testing.T) { } m.mu.Lock() entry := *m.catalog.entries["data"] - entry.tenants = []string{"different-tenant"} + entry.acl = newVolumeACL([]string{"different-tenant"}, volumePublic) m.catalog.entries["data"] = &entry m.mu.Unlock() if err := os.Remove(tmp); err != nil { From 83c17414b1ff0c5b949db0f9aac7538eb618def8 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:53:52 +0000 Subject: [PATCH 37/72] sdk: centralize volume response decoding --- docs/sandboxd-api.md | 4 ++++ sandboxd/server/server_http.go | 15 ++++++++----- sdk/go/client.go | 5 +++-- sdk/go/info.go | 4 ---- sdk/go/volume_mutation_test.go | 33 ++++++++++++++++++++++------ sdk/go/volumes.go | 23 +++++++++++-------- sdk/python/cocoonsandbox/client.py | 21 +++++++++--------- sdk/python/cocoonsandbox/template.py | 1 + sdk/python/cocoonsandbox/volumes.py | 30 +++++++++++++++++++++++-- sdk/python/tests/test_client.py | 27 ++++++++++++++++++++++- 10 files changed, 122 insertions(+), 41 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index e381f648..08307ebe 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -257,6 +257,10 @@ before changing devices. The current source Go and Python SDKs send all-attach-only initial claims using the existing `volumes_attach_only` flag without per-entry flags. They reject mixed mounted and attach-only initial claims locally, before network I/O. +That restriction can be removed when the minimum supported server version +honors per-entry `attach_only` on initial claims at every possible placement +and redirect target. Until then, the SDK cannot know that a redirected claim +will preserve the mixed mounting choice. Claim without volumes, then use PUT or PATCH for a mixed live list. These SDK changes are unreleased. The published Python 0.1.19 SDK accepts only `name`, `mount`, and `mode` in request mappings. HTTP responses retain those fields, diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index 0704299e..1201fb97 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -22,14 +22,17 @@ import ( const retryAfterSeconds = "10" -// Keep HTTP responses readable by SDKs that echo the historical three-field shape. +// Keep HTTP responses readable by SDKs that echo the historical volume shape. // Storage and forwarded requests retain the canonical AttachOnly field. +type volumeView struct { + Name string `json:"name"` + Mount string `json:"mount,omitempty"` + Mode string `json:"mode,omitempty"` + AttachOnly bool `json:"-"` +} + var responseMarshalers = json.MarshalToFunc(func(out *jsontext.Encoder, v types.Volume) error { - return json.MarshalEncode(out, struct { - Name string `json:"name"` - Mount string `json:"mount,omitempty"` - Mode string `json:"mode,omitempty"` - }{v.Name, v.Mount, v.Mode}) + return json.MarshalEncode(out, volumeView(v)) }) type tenantKey struct{} diff --git a/sdk/go/client.go b/sdk/go/client.go index 70bf2632..b94eb753 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -124,6 +124,7 @@ func (r *claimRequest) prepareVolumes() error { } } if !r.VolumesAttachOnly && attachOnly > 0 && attachOnly != len(r.Volumes) { + // Remove this guard only when every supported claim and redirect target honors per-entry attach_only. return errors.New("mixed mounted and attach-only initial claims are not safe on older servers: claim without volumes, then use SetVolumes or PatchVolumes") } if attachOnly > 0 { @@ -266,7 +267,7 @@ func (c *Client) ownerAt(ctx context.Context, addr, id, token string) (string, e } func (c *Client) handleFrom(dialed string, cr claimResponse) *Sandbox { - volumes := responseVolumes(cr.Volumes) + volumes := slices.Clone(cr.Volumes) return &Sandbox{ ID: cr.ID, Deadline: cr.Deadline, Volumes: volumes, VolumesWant: slices.Clone(volumes), FromCheckpoint: cr.FromCheckpoint, TemplateDigest: cr.TemplateDigest, NetRoute: cr.NetRoute, @@ -329,7 +330,7 @@ func (c *Client) doJSON[T any](ctx context.Context, method, addr, path string, b if resp.StatusCode != http.StatusOK { return out, apiError(verb, resp) } - if err := json.UnmarshalRead(resp.Body, &out); err != nil { + if err := json.UnmarshalRead(resp.Body, &out, json.WithUnmarshalers(responseUnmarshalers)); err != nil { return out, fmt.Errorf("decode %s response: %w", verb, err) } return out, nil diff --git a/sdk/go/info.go b/sdk/go/info.go index 3a08e819..86879cd3 100644 --- a/sdk/go/info.go +++ b/sdk/go/info.go @@ -90,10 +90,6 @@ func (c *Client) Sandboxes(ctx context.Context) ([]SandboxSummary, error) { if err != nil { return nil, err } - for i := range reply.Sandboxes { - reply.Sandboxes[i].Volumes = responseVolumes(reply.Sandboxes[i].Volumes) - reply.Sandboxes[i].VolumesWant = responseVolumes(reply.Sandboxes[i].VolumesWant) - } return reply.Sandboxes, nil } diff --git a/sdk/go/volume_mutation_test.go b/sdk/go/volume_mutation_test.go index 3b06f079..3c258326 100644 --- a/sdk/go/volume_mutation_test.go +++ b/sdk/go/volume_mutation_test.go @@ -270,18 +270,37 @@ func TestVolumeResponseInferenceDoesNotChangeRequestDecoding(t *testing.T) { } } -func TestClaimResponseVolumesAreOwnedCopies(t *testing.T) { - c := &Client{} - wire := []Volume{{Name: "raw"}, {Name: "mounted", Mount: "/data"}} - sb := c.handleFrom("node", claimResponse{Volumes: wire}) - if !sb.Volumes[0].AttachOnly || sb.Volumes[1].AttachOnly || wire[0].AttachOnly { - t.Fatalf("response inference changed wire data: handle=%+v wire=%+v", sb.Volumes, wire) +func TestClaimResponseDecoderAndOwnedCopies(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"id":"sb_1","token":"tok","volumes":[{"name":"raw"},{"name":"mounted","mount":"/data"}]}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + wire, err := c.doJSON[claimResponse](t.Context(), http.MethodPost, c.addr, "/v1/claim", nil, "", "claim") + if err != nil { + t.Fatal(err) + } + sb := c.handleFrom("node", wire) + if !sb.Volumes[0].AttachOnly || sb.Volumes[1].AttachOnly || !wire.Volumes[0].AttachOnly { + t.Fatalf("response decode lost attach-only state: handle=%+v wire=%+v", sb.Volumes, wire) } sb.Volumes[0].Name = "caller" - if wire[0].Name != "raw" || sb.VolumesWant[0].Name != "raw" { + if wire.Volumes[0].Name != "raw" || sb.VolumesWant[0].Name != "raw" { t.Fatal("claim volume views alias the wire or each other") } if c.handleFrom("node", claimResponse{}).Volumes != nil || c.handleFrom("node", claimResponse{Volumes: []Volume{}}).Volumes == nil { t.Fatal("claim response changed nil versus empty volume semantics") } } + +func TestForkResponseDecodesNestedVolumes(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"children":[{"id":"sb_child","token":"tok","volumes":[{"name":"raw"},{"name":"mounted","mount":"/data"}]}]}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + reply, err := c.doJSON[forkResponse](t.Context(), http.MethodPost, c.addr, "/v1/sandboxes/sb_1/fork", nil, "", "fork") + if err != nil || len(reply.Children) != 1 || !reply.Children[0].Volumes[0].AttachOnly || reply.Children[0].Volumes[1].AttachOnly { + t.Fatalf("nested fork volumes did not decode: %+v err=%v", reply, err) + } +} diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go index 24b0310f..d54a1404 100644 --- a/sdk/go/volumes.go +++ b/sdk/go/volumes.go @@ -3,6 +3,7 @@ package sandbox import ( "bytes" "context" + "encoding/json/jsontext" "encoding/json/v2" "errors" "fmt" @@ -91,14 +92,18 @@ func validateVolume(v Volume) error { } // Historical responses identify attach-only devices by their absent mount. -// Keep that inference out of request decoding, where an absent mount means the default path. -func responseVolumes(volumes []Volume) []Volume { - volumes = slices.Clone(volumes) - for i := range volumes { - volumes[i].AttachOnly = volumes[i].Mount == "" +// Requests use the same public type, so this decoder applies only to network replies. +type responseVolume Volume + +var responseUnmarshalers = json.UnmarshalFromFunc(func(in *jsontext.Decoder, v *Volume) error { + var wire responseVolume + if err := json.UnmarshalDecode(in, &wire); err != nil { + return err } - return volumes -} + *v = Volume(wire) + v.AttachOnly = v.Mount == "" + return nil +}) func (s *Sandbox) requestVolumes(ctx context.Context, method string, request any, verb string) (SandboxVolumesResponse, error) { var out SandboxVolumesResponse @@ -122,10 +127,10 @@ func (s *Sandbox) requestVolumes(ctx context.Context, method string, request any Converging *bool `json:"converging"` Error string `json:"error"` } - decodeErr := json.UnmarshalRead(resp.Body, &wire) + decodeErr := json.UnmarshalRead(resp.Body, &wire, json.WithUnmarshalers(responseUnmarshalers)) complete := decodeErr == nil && wire.VolumesWant != nil && wire.Volumes != nil && wire.States != nil && wire.Converging != nil if complete { - out = SandboxVolumesResponse{VolumesWant: responseVolumes(*wire.VolumesWant), Volumes: responseVolumes(*wire.Volumes), States: *wire.States, Converging: *wire.Converging, Error: wire.Error} + out = SandboxVolumesResponse{VolumesWant: *wire.VolumesWant, Volumes: *wire.Volumes, States: *wire.States, Converging: *wire.Converging, Error: wire.Error} s.VolumesWant = slices.Clone(out.VolumesWant) s.Volumes = slices.Clone(out.Volumes) } diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 4e8be349..2565f459 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -18,7 +18,7 @@ from .errors import APIError, SandboxTimeout, require_field from .sandbox import Sandbox from .template import _claim_body, _template_query -from .volumes import Volume, _response_volumes +from .volumes import Volume, _normalize_volume_reply T = TypeVar("T") @@ -110,12 +110,7 @@ def attach(self, owner_addr: str, id: str, token: str) -> Sandbox: def sandboxes(self) -> list[dict[str, Any]]: """Lists the claims this token may see: id, key, deadline, claim_ref — never tokens or host paths.""" reply = self._request(self.addr, "GET", "/v1/sandboxes", None, "list sandboxes") - rows = [dict(sb) for sb in reply.get("sandboxes") or []] - for row in rows: - for key in ("volumes", "volumes_want"): - if row.get(key) is not None: - row[key] = _response_volumes(row[key]) - return rows + return [dict(sb) for sb in reply.get("sandboxes") or []] def drain(self) -> dict[str, Any]: """Cordons the node (root token): new claims are refused, live ones run to their leases.""" @@ -186,7 +181,7 @@ def _handle_from(self, dialed: str, reply: dict[str, Any], verb: str = "claim") deadline=reply.get("deadline", ""), from_checkpoint=reply.get("from_checkpoint", ""), template_digest=reply.get("template_digest", ""), - volumes=_response_volumes(reply.get("volumes") or []), + volumes=reply.get("volumes") or [], ) def _post_json( @@ -223,11 +218,17 @@ def _request( reply = json.loads(raw) except (ValueError, TypeError): reply = None - raise APIError(verb, status, _error_message(raw), reply=reply if isinstance(reply, dict) else None) + raise APIError( + verb, + status, + _error_message(raw), + reply=_normalize_volume_reply(path, reply) if isinstance(reply, dict) else None, + ) if not raw: return {} try: - return cast(dict[str, Any], json.loads(raw)) + reply = json.loads(raw) + return cast(dict[str, Any], _normalize_volume_reply(path, reply) if isinstance(reply, dict) else reply) except json.JSONDecodeError as exc: raise APIError(verb, 0, "malformed JSON in response") from exc diff --git a/sdk/python/cocoonsandbox/template.py b/sdk/python/cocoonsandbox/template.py index 9e809e81..d477679a 100644 --- a/sdk/python/cocoonsandbox/template.py +++ b/sdk/python/cocoonsandbox/template.py @@ -70,6 +70,7 @@ def _claim_body( entries = _volume_bodies(volumes, mount) raw = sum(bool(entry.get("attach_only")) for entry in entries) if mount and raw and raw != len(entries): + # Remove this guard only when every supported claim and redirect target honors per-entry attach_only. raise TypeError( "mixed mounted and attach-only initial claims are not safe on older servers: " "claim without volumes, then use set_volumes or patch_volumes" diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index a0a44a9b..ce025d06 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -55,12 +55,38 @@ def _volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | N ): return None view = dict(reply) - view["volumes_want"] = _response_volumes(reply["volumes_want"]) - view["volumes"] = _response_volumes(reply["volumes"]) + view["volumes_want"] = [dict(volume) for volume in reply["volumes_want"]] + view["volumes"] = [dict(volume) for volume in reply["volumes"]] view["states"] = [dict(state) for state in reply["states"]] return cast(SandboxVolumesResponse, view) +def _normalize_volume_reply(path: str, reply: dict[str, Any]) -> dict[str, Any]: + """Interpret mountless volume entries only in known HTTP response containers.""" + route = path.partition("?")[0].split("/") + if route == ["", "v1", "claim"] or ( + len(route) == 5 and route[1:3] == ["v1", "checkpoints"] and route[4] == "claim" + ): + _normalize_volume_field(reply, "volumes") + elif route == ["", "v1", "sandboxes"]: + for row in reply.get("sandboxes") or []: + _normalize_volume_field(row, "volumes") + _normalize_volume_field(row, "volumes_want") + elif len(route) == 5 and route[1:3] == ["v1", "sandboxes"]: + if route[4] == "fork": + for child in reply.get("children") or []: + _normalize_volume_field(child, "volumes") + elif route[4] == "volumes": + _normalize_volume_field(reply, "volumes") + _normalize_volume_field(reply, "volumes_want") + return reply + + +def _normalize_volume_field(container: dict[str, Any], key: str) -> None: + if isinstance(container.get(key), list): + container[key] = _response_volumes(container[key]) + + def _response_volumes(volumes: Iterable[Mapping[str, str | bool] | Volume]) -> list[Volume]: # Response mounts identify raw devices; absent request mounts instead select a default path. entries = [cast("Volume", dict(volume)) for volume in volumes] diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 73de4e74..c7f6c8ff 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -503,7 +503,8 @@ def generate(): def test_claim_and_summary_response_inference_copies_wire_data(node): client = Client(node) reply = {"id": "sb_1", "token": "tok", "volumes": [{"name": "raw"}, {"name": "mounted", "mount": "/data"}]} - sb = client._handle_from(node, reply) + FakeNode.routes[("POST", "/v1/claim")] = lambda body, path: (200, reply) + sb = client.new("rt:24.04") assert sb.volumes[0]["attach_only"] is True assert "attach_only" not in sb.volumes[1] assert "attach_only" not in reply["volumes"][0] @@ -521,3 +522,27 @@ def test_claim_and_summary_response_inference_copies_wire_data(node): assert summaries[1]["volumes"] == summaries[1]["volumes_want"] == [] assert "volumes" not in summaries[2] assert "attach_only" not in rows[0]["volumes"][0] + + +@pytest.mark.parametrize( + ("path", "payload", "expected"), + [ + ("/v1/checkpoints/ck_1/claim", {"volumes": [{"name": "raw"}]}, True), + ("/v1/sandboxes/sb_1/fork", {"children": [{"volumes": [{"name": "raw"}]}]}, True), + ("/v1/volumes", {"volumes": [{"name": "catalog"}]}, False), + ("/v1/info", {"metadata": {"volumes": [{"name": "opaque"}]}}, False), + ], +) +def test_response_decoder_only_interprets_volume_bearing_routes(spawn_node, path, payload, expected): + addr = spawn_node({("GET", path): lambda body, request_path: (200, payload)}) + reply = Client(addr)._request(addr, "GET", path, None, "inspect") + if path.endswith("/claim"): + entry = reply["volumes"][0] + elif path.endswith("/fork"): + entry = reply["children"][0]["volumes"][0] + elif path == "/v1/volumes": + entry = reply["volumes"][0] + else: + entry = reply["metadata"]["volumes"][0] + assert (entry.get("attach_only") is True) is expected + assert "attach_only" not in repr(payload) From bbb51214384ae58aab980513b876a1ea1dd8d2e4 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:56:12 +0000 Subject: [PATCH 38/72] pool: reject access through an unset volume policy --- sandboxd/pool/volume_acl_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go index d08b9cbc..8d66ded1 100644 --- a/sandboxd/pool/volume_acl_test.go +++ b/sandboxd/pool/volume_acl_test.go @@ -10,6 +10,17 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +func TestUnsetVolumeACLRejectsAccess(t *testing.T) { + var acl volumeACL + for _, caller := range []string{"", "acme"} { + for _, owner := range []string{"", "acme"} { + if acl.allowedFor(volumeAccessFor(caller, owner)) { + t.Errorf("unset volume ACL allowed caller %q for owner %q", caller, owner) + } + } + } +} + func TestRuntimeCatalogDefaultACL(t *testing.T) { requireCatalogRoot(t) for _, tenants := range [][]string{nil, {}} { From 0b4415314746d38d93a6df577db8a498ecbf3f7b Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 04:56:24 +0000 Subject: [PATCH 39/72] pool: leave unset volume access policy invalid --- sandboxd/pool/volume.go | 2 +- sdk/go/volumes.go | 1 - sdk/python/cocoonsandbox/volumes.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 407ce314..e34d2343 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -56,7 +56,7 @@ type catalogVolume struct { type volumeACLMode uint8 const ( - volumeOperatorOnly volumeACLMode = iota + volumeOperatorOnly volumeACLMode = iota + 1 volumePublic volumeTenantList ) diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go index d54a1404..a632cc70 100644 --- a/sdk/go/volumes.go +++ b/sdk/go/volumes.go @@ -92,7 +92,6 @@ func validateVolume(v Volume) error { } // Historical responses identify attach-only devices by their absent mount. -// Requests use the same public type, so this decoder applies only to network replies. type responseVolume Volume var responseUnmarshalers = json.UnmarshalFromFunc(func(in *jsontext.Decoder, v *Volume) error { diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index ce025d06..9bda0fab 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -62,7 +62,6 @@ def _volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | N def _normalize_volume_reply(path: str, reply: dict[str, Any]) -> dict[str, Any]: - """Interpret mountless volume entries only in known HTTP response containers.""" route = path.partition("?")[0].split("/") if route == ["", "v1", "claim"] or ( len(route) == 5 and route[1:3] == ["v1", "checkpoints"] and route[4] == "claim" From 2f96207bc2e3e25fe9a403ca413a3c3667cc4698 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 13:24:27 +0800 Subject: [PATCH 40/72] pool: cover legacy attach-only PATCH recovery --- sandboxd/pool/volume_modes_test.go | 85 ++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/sandboxd/pool/volume_modes_test.go b/sandboxd/pool/volume_modes_test.go index 282e4ed6..599ca6c0 100644 --- a/sandboxd/pool/volume_modes_test.go +++ b/sandboxd/pool/volume_modes_test.go @@ -85,6 +85,91 @@ func TestLiveVolumeFourModesAndMixedRoundtrip(t *testing.T) { } } +func TestLiveVolumeLegacyAttachOnlyRetryAfterFailedDetach(t *testing.T) { + for _, mode := range []string{"", types.VolumeModeRW} { + for _, method := range []string{"put", "patch"} { + t.Run(fmt.Sprintf("%s/%s", map[string]string{"": "ro", types.VolumeModeRW: "rw"}[mode], method), func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + want := []types.Volume{{Name: "data", Mode: mode, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatal(err) + } + eng.detachVolumeErr = errors.New("host detach unavailable") + view, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + if err == nil || len(view.VolumesWant) != 0 || !slices.Equal(view.Volumes, want) { + t.Fatalf("failed detach lost applied device: %+v, %v", view, err) + } + eng.detachVolumeErr = nil + before := slices.Clone(eng.volumeOpsLog()) + change := func(volumes []types.Volume) (types.SandboxVolumesResponse, error) { + if method == "patch" { + return m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: volumes}) + } + return m.SetVolumes(t.Context(), sb.ID, "", volumes) + } + if _, err = change([]types.Volume{{Name: "data", Mode: mode, Mount: "/changed"}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("held attachment changed shape: %v", err) + } + if !slices.Equal(before, eng.volumeOpsLog()) { + t.Fatal("rejected shape change operated on the device") + } + legacy := slices.Clone(want) + legacy[0].AttachOnly = false + view, err = change(legacy) + if err != nil { + t.Fatalf("legacy echo of held attach-only volume rejected: %v", err) + } + if !slices.Equal(view.VolumesWant, want) || !slices.Equal(view.Volumes, want) || view.Converging { + t.Fatalf("legacy retry did not retain the applied device: %+v", view) + } + before = slices.Clone(eng.volumeOpsLog()) + if _, err = change(legacy); err != nil { + t.Fatalf("settled legacy retry: %v", err) + } + if !slices.Equal(before, eng.volumeOpsLog()) { + t.Fatalf("settled retry changed the retained device: %v -> %v", before, eng.volumeOpsLog()) + } + }) + } + } +} + +func TestLiveVolumeLegacyAttachOnlyPatchAfterFailedCommit(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + want := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || !view.Converging { + t.Fatalf("failed detach commit did not retain a recovery reference: %+v, %v", view, err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + legacy := []types.Volume{{Name: "data", Mode: types.VolumeModeRW}} + view, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: legacy}) + if err != nil { + t.Fatalf("legacy echo of recovery-only attach-only volume rejected: %v", err) + } + if !slices.Equal(view.VolumesWant, want) || !slices.Equal(view.Volumes, want) || view.Converging { + t.Fatalf("legacy retry lost recovery attachment shape: %+v", view) + } +} + func TestLiveWritableDetachRequiresCleanEvidence(t *testing.T) { for _, proof := range []bool{false, true} { t.Run(fmt.Sprint(proof), func(t *testing.T) { From 1e3a6f7637fec570fb5b7321718e19f878e608b1 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 13:31:09 +0800 Subject: [PATCH 41/72] pool: preserve held volume shape during PATCH recovery --- docs/sdk.md | 5 +++-- sandboxd/pool/volume_mutation.go | 15 ++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/sdk.md b/docs/sdk.md index 75e3175b..670f549a 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -288,8 +288,9 @@ Changing a held name's mount, mode, or mounted shape requires a successful remove before adding it again. Paused sandboxes are not woken, and images must exist on the owner node. Admission holds cover the union of desired and applied sets. A settled retained entry receives no new attach or mount calls. `converging` -is true only during an active request. Pending or detaching entries with false -require an explicit retry. Daemon restart replaces unfinished diagnostics with +is true while a request worker is running or a recovery-only reference awaits +an acknowledged claim write. Pending or detaching entries with false require +an explicit retry. Daemon restart replaces unfinished diagnostics with an interrupted message. Capture remains refused while either set or unacknowledged recovery references hold volumes. Release cancels convergence and hands held volumes to teardown. See the diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 03e6a7d2..74d61097 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -18,7 +18,7 @@ const ( volumeChangeInterrupted = "volume change interrupted; retry the desired target" ) -type volumeEdit func([]types.Volume) ([]types.Volume, error) +type volumeEdit func(current, shapeContext []types.Volume) ([]types.Volume, error) type volumeRun struct { cancel context.CancelFunc @@ -40,16 +40,16 @@ func (m *Manager) GetVolumes(id, tenant string) (types.SandboxVolumesResponse, e // SetVolumes converges a running claim onto the full desired volume set. func (m *Manager) SetVolumes(ctx context.Context, id, tenant string, want []types.Volume) (types.SandboxVolumesResponse, error) { - return m.changeVolumes(ctx, id, tenant, func([]types.Volume) ([]types.Volume, error) { return slices.Clone(want), nil }) + return m.changeVolumes(ctx, id, tenant, func([]types.Volume, []types.Volume) ([]types.Volume, error) { return slices.Clone(want), nil }) } // PatchVolumes adds and removes a running claim's volumes by name. func (m *Manager) PatchVolumes(ctx context.Context, id, tenant string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { - return m.changeVolumes(ctx, id, tenant, func(current []types.Volume) ([]types.Volume, error) { + return m.changeVolumes(ctx, id, tenant, func(current, shapeContext []types.Volume) ([]types.Volume, error) { if err := validateDetachNames(patch.Remove); err != nil { return nil, err } - additions, err := normalizeLiveVolumes(patch.Add, current) + additions, err := normalizeLiveVolumes(patch.Add, shapeContext) if err != nil { return nil, err } @@ -91,15 +91,16 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v current, held := slices.Clone(sb.VolumesWant), heldVolumes(sb) recovery := m.store.recoveryVolumes(sb.ID)[sb.ID] m.mu.Unlock() - target, err := change(current) + shapeContext := slices.Concat(held, recovery) + target, err := change(current, shapeContext) if err != nil { return view, err } - target, err = normalizeLiveVolumes(target, slices.Concat(held, recovery)) + target, err = normalizeLiveVolumes(target, shapeContext) if err != nil { return view, err } - if err = validateLiveTarget(slices.Concat(held, recovery), target); err != nil { + if err = validateLiveTarget(shapeContext, target); err != nil { return view, err } additions := volumeDifferenceByName(target, held) From ace99b33c3d3df70f80f08a9964386c7dd9da6c2 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 05:39:48 +0000 Subject: [PATCH 42/72] sdk/python: reproduce volume response decoding failures --- sdk/python/tests/test_client.py | 28 ++++++++++++++++++++++++++++ sdk/python/tests/test_volumes.py | 23 +++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index c7f6c8ff..0602ca42 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -546,3 +546,31 @@ def test_response_decoder_only_interprets_volume_bearing_routes(spawn_node, path entry = reply["metadata"]["volumes"][0] assert (entry.get("attach_only") is True) is expected assert "attach_only" not in repr(payload) + + +def test_claim_response_decoding_does_not_depend_on_route(spawn_node): + path = "/v1/custom-claims" + payload = {"id": "sb_1", "token": "tok", "volumes": [{"name": "raw"}]} + addr = spawn_node({("POST", path): lambda body, request_path: (200, payload)}) + sb = Client(addr)._claim_from(addr, {}, path=path) + assert sb.volumes == [{"name": "raw", "attach_only": True}] + assert "attach_only" not in payload["volumes"][0] + + +@pytest.mark.parametrize( + ("path", "fields"), + [ + ("/v1/claim", {"volumes": [None]}), + ("/v1/sandboxes", {"sandboxes": [None]}), + ("/v1/sandboxes/sb_1/fork", {"children": [None]}), + ("/v1/sandboxes/sb_1/volumes", {"volumes": ["bad"]}), + ], +) +def test_malformed_error_details_do_not_hide_http_error(spawn_node, path, fields): + payload = {"error": "storage unavailable", **fields} + addr = spawn_node({("GET", path): lambda body, request_path: (503, payload)}) + with pytest.raises(APIError) as caught: + Client(addr)._request(addr, "GET", path, None, "inspect") + assert caught.value.status == 503 + assert caught.value.message == "storage unavailable" + assert caught.value.reply == payload diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 59cc4a9e..38bf4a43 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -292,3 +292,26 @@ def generate(): entries = [] if kind == "empty-generator" else [{"name": "data"}] assert seen == [{"volumes" if method == "set" else "add": entries}] assert consumed == (["data"] if kind == "generator" else []) + + +@pytest.mark.parametrize("status", [200, 409]) +@pytest.mark.parametrize("field", ["volumes", "volumes_want", "states"]) +@pytest.mark.parametrize("entry", [None, "bad", []]) +def test_malformed_volume_view_preserves_error_and_handle(spawn_node, status, field, entry): + payload = {**VIEW, "error": "storage unavailable", field: [*VIEW[field], entry]} + sb = sandbox_at(spawn_node({("GET", PATH): lambda body, path: (status, payload)})) + sb.volumes = [{"name": "old"}] + sb.volumes_want = [{"name": "wanted"}] + with pytest.raises(APIError) as caught: + sb.get_volumes() + assert sb.volumes == [{"name": "old"}] + assert sb.volumes_want == [{"name": "wanted"}] + if status == 409: + assert isinstance(caught.value, VolumeError) + assert caught.value.status == 409 + assert caught.value.message == "storage unavailable" + assert caught.value.reply == payload + assert caught.value.response is None + else: + assert caught.value.status == 0 + assert caught.value.message == "reply without complete volume view" From 00fff735d32a82d8158f39ed7be297924ea62140 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 05:41:21 +0000 Subject: [PATCH 43/72] sdk/python: decode volumes at response boundaries --- sdk/python/cocoonsandbox/client.py | 8 +++--- sdk/python/cocoonsandbox/sandbox.py | 4 +-- sdk/python/cocoonsandbox/volumes.py | 40 +++++++++------------------- sdk/python/tests/test_client.py | 41 ++++++++++++++++++----------- 4 files changed, 43 insertions(+), 50 deletions(-) diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 2565f459..18f3f945 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -18,7 +18,7 @@ from .errors import APIError, SandboxTimeout, require_field from .sandbox import Sandbox from .template import _claim_body, _template_query -from .volumes import Volume, _normalize_volume_reply +from .volumes import Volume, _sandbox_response T = TypeVar("T") @@ -110,7 +110,7 @@ def attach(self, owner_addr: str, id: str, token: str) -> Sandbox: def sandboxes(self) -> list[dict[str, Any]]: """Lists the claims this token may see: id, key, deadline, claim_ref — never tokens or host paths.""" reply = self._request(self.addr, "GET", "/v1/sandboxes", None, "list sandboxes") - return [dict(sb) for sb in reply.get("sandboxes") or []] + return [_sandbox_response(sb) for sb in reply.get("sandboxes") or []] def drain(self) -> dict[str, Any]: """Cordons the node (root token): new claims are refused, live ones run to their leases.""" @@ -222,13 +222,13 @@ def _request( verb, status, _error_message(raw), - reply=_normalize_volume_reply(path, reply) if isinstance(reply, dict) else None, + reply=reply if isinstance(reply, dict) else None, ) if not raw: return {} try: reply = json.loads(raw) - return cast(dict[str, Any], _normalize_volume_reply(path, reply) if isinstance(reply, dict) else reply) + return cast(dict[str, Any], reply) except json.JSONDecodeError as exc: raise APIError(verb, 0, "malformed JSON in response") from exc diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 4981a222..46cdd393 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -24,7 +24,7 @@ ) from .frames import BULK_CHUNK, FS_CHUNK, KEEP_ALIVE_PROTO from .template import Template -from .volumes import SandboxVolumesResponse, Volume, _volume_bodies, _volume_response +from .volumes import SandboxVolumesResponse, Volume, _response_volumes, _volume_bodies, _volume_response if TYPE_CHECKING: from .client import Client @@ -54,7 +54,7 @@ def __init__( self.deadline = deadline self.from_checkpoint = from_checkpoint self.template_digest = template_digest - self.volumes = [dict(cast("Mapping[str, str | bool]", volume)) for volume in volumes or []] + self.volumes = [cast("dict[str, str | bool]", volume) for volume in _response_volumes(volumes or [])] self.volumes_want = [dict(volume) for volume in self.volumes] self._pool = ConnPool(client.keep_alive) self._proto = 0 diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index 9bda0fab..7cca6cac 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -49,41 +49,25 @@ class SandboxVolumesResponse(_SandboxVolumesResponse, total=False): def _volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | None: if ( - reply is None - or not all(isinstance(reply.get(key), list) for key in ("volumes_want", "volumes", "states")) + not isinstance(reply, dict) + or not all( + isinstance(reply.get(key), list) and all(isinstance(entry, dict) for entry in reply[key]) + for key in ("volumes_want", "volumes", "states") + ) or not isinstance(reply.get("converging"), bool) ): return None - view = dict(reply) - view["volumes_want"] = [dict(volume) for volume in reply["volumes_want"]] - view["volumes"] = [dict(volume) for volume in reply["volumes"]] + view = _sandbox_response(reply) view["states"] = [dict(state) for state in reply["states"]] return cast(SandboxVolumesResponse, view) -def _normalize_volume_reply(path: str, reply: dict[str, Any]) -> dict[str, Any]: - route = path.partition("?")[0].split("/") - if route == ["", "v1", "claim"] or ( - len(route) == 5 and route[1:3] == ["v1", "checkpoints"] and route[4] == "claim" - ): - _normalize_volume_field(reply, "volumes") - elif route == ["", "v1", "sandboxes"]: - for row in reply.get("sandboxes") or []: - _normalize_volume_field(row, "volumes") - _normalize_volume_field(row, "volumes_want") - elif len(route) == 5 and route[1:3] == ["v1", "sandboxes"]: - if route[4] == "fork": - for child in reply.get("children") or []: - _normalize_volume_field(child, "volumes") - elif route[4] == "volumes": - _normalize_volume_field(reply, "volumes") - _normalize_volume_field(reply, "volumes_want") - return reply - - -def _normalize_volume_field(container: dict[str, Any], key: str) -> None: - if isinstance(container.get(key), list): - container[key] = _response_volumes(container[key]) +def _sandbox_response(reply: dict[str, Any]) -> dict[str, Any]: + view = dict(reply) + for key in ("volumes", "volumes_want"): + if isinstance(reply.get(key), list): + view[key] = _response_volumes(reply[key]) + return view def _response_volumes(volumes: Iterable[Mapping[str, str | bool] | Volume]) -> list[Volume]: diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 0602ca42..024cd9fb 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -525,27 +525,19 @@ def test_claim_and_summary_response_inference_copies_wire_data(node): @pytest.mark.parametrize( - ("path", "payload", "expected"), + ("path", "payload"), [ - ("/v1/checkpoints/ck_1/claim", {"volumes": [{"name": "raw"}]}, True), - ("/v1/sandboxes/sb_1/fork", {"children": [{"volumes": [{"name": "raw"}]}]}, True), - ("/v1/volumes", {"volumes": [{"name": "catalog"}]}, False), - ("/v1/info", {"metadata": {"volumes": [{"name": "opaque"}]}}, False), + ("/v1/checkpoints/ck_1/claim", {"volumes": [{"name": "raw"}]}), + ("/v1/sandboxes/sb_1/fork", {"children": [{"volumes": [{"name": "raw"}]}]}), + ("/v1/volumes", {"volumes": [{"name": "catalog"}]}), + ("/v1/info", {"metadata": {"volumes": [{"name": "opaque"}]}}), ], ) -def test_response_decoder_only_interprets_volume_bearing_routes(spawn_node, path, payload, expected): +def test_transport_keeps_json_uninterpreted(spawn_node, path, payload): addr = spawn_node({("GET", path): lambda body, request_path: (200, payload)}) reply = Client(addr)._request(addr, "GET", path, None, "inspect") - if path.endswith("/claim"): - entry = reply["volumes"][0] - elif path.endswith("/fork"): - entry = reply["children"][0]["volumes"][0] - elif path == "/v1/volumes": - entry = reply["volumes"][0] - else: - entry = reply["metadata"]["volumes"][0] - assert (entry.get("attach_only") is True) is expected - assert "attach_only" not in repr(payload) + assert reply == payload + assert "attach_only" not in repr(reply) def test_claim_response_decoding_does_not_depend_on_route(spawn_node): @@ -574,3 +566,20 @@ def test_malformed_error_details_do_not_hide_http_error(spawn_node, path, fields assert caught.value.status == 503 assert caught.value.message == "storage unavailable" assert caught.value.reply == payload + + +@pytest.mark.parametrize("operation", ["checkpoint", "fork"]) +def test_sandbox_handle_decodes_volume_responses(spawn_node, operation): + payload = {"id": "sb_2", "token": "tok", "volumes": [{"name": "raw"}, {"name": "mounted", "mount": "/data"}]} + if operation == "checkpoint": + path = "/v1/checkpoints/ck_1/claim" + response = payload + else: + path = "/v1/sandboxes/sb_1/fork" + response = {"children": [payload]} + addr = spawn_node({("POST", path): lambda body, request_path: (200, response)}) + client = Client(addr) + sb = client.checkpoint("ck_1").new() if operation == "checkpoint" else sandbox_at(addr).fork(1)[0] + assert sb.volumes == [{"name": "raw", "attach_only": True}, {"name": "mounted", "mount": "/data"}] + assert sb.volumes_want == sb.volumes + assert "attach_only" not in payload["volumes"][0] From 3e6678e1296e6f71ade54fe43880481791238b6c Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 05:42:40 +0000 Subject: [PATCH 44/72] docs: test the external volume controller example --- docs/live-volumes.md | 7 +- docs/sandboxd-api.md | 16 ++- docs/security.md | 7 +- docs/volume-controller.md | 83 ++++++------- sdk/python/examples/volume_controller.py | 37 ++++++ sdk/python/tests/test_volume_controller.py | 138 +++++++++++++++++++++ 6 files changed, 234 insertions(+), 54 deletions(-) create mode 100644 sdk/python/examples/volume_controller.py create mode 100644 sdk/python/tests/test_volume_controller.py diff --git a/docs/live-volumes.md b/docs/live-volumes.md index af4044c8..82e798cd 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -18,9 +18,10 @@ Keep `api_token` configured. List tenants explicitly to grant self-service; that permission also allows reattachment to another claim after detach. For a sleep and wake workflow, an external controller owns quiescence and unmount before detach, a reservation through sleep, and access gating until wake, attach, and mount complete. That -gate also covers business requests that wake the sandbox. Wake establishes -only silkd readiness. The API does not include a controller implementation or -verify that composed workflow. Live changes support mounted and attach-only +gate covers API and preview access on every node that can wake the sandbox. +Wake establishes only silkd readiness. The [controller example](volume-controller.md) +has SDK and HTTP contract tests. The controller still supplies the reservation, +guest operations, and access gate. Live changes support mounted and attach-only entries in read-only or writable mode. Pause application reads before replacement. Close files, mmap mappings, cwd, diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 08307ebe..57771b2b 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -369,16 +369,22 @@ has retired. An external controller owns quiescence and guest unmount before detach. It confirms successful detach and recovery bookkeeping before hibernation, and -keeps its own volume reservation while the sandbox sleeps. For controller-only attachment, leave the runtime entry's `tenants` empty and -keep `api_token` configured. A nonempty ACL allows those tenants to attach the +keeps its own volume reservation while the sandbox sleeps. sandboxd holds no +reservation after detach and reference retirement, so root can attach that +volume elsewhere during sleep. For controller-only attachment, leave the runtime +entry's `tenants` empty and keep `api_token` configured. A nonempty ACL allows those tenants to attach the volume themselves after detach; it does not reserve the volume for one claim. Removing `api_token` restores the daemon's unauthenticated root-scope mode, so operator-only admission then provides no network authentication. The controller -gates every access path, including business requests that wake the sandbox, -until wake, attach, and mount complete. Wake establishes only silkd readiness. +gates every access path, including the API and preview listeners on all nodes +that can serve or wake the sandbox, until wake, attach, and mount complete. +Wake establishes only silkd readiness. It does not restore volume attachments or application readiness. Controller orchestration is external. The [controller example](volume-controller.md) -shows the sequence and the reservation and access-gate contracts it requires. +shows the sequence, the pool replication and archive settings, and the +reservation and access-gate contracts it requires. A change of owning node +requires the controller to fence the old node before attaching the same backing +on a new node, even when the claim ID is unchanged. Downgrading to a daemon without runtime registration requires draining runtime references or supplying equivalent static volume config. Pending live volume changes must finish before downgrade. diff --git a/docs/security.md b/docs/security.md index 12382fc8..2861207e 100644 --- a/docs/security.md +++ b/docs/security.md @@ -170,8 +170,11 @@ before read-only publication. An external controller owns stop-write and unmount before detach, its own reservation across sleep, and access gating until wake, attach, and mount complete. Wake establishes silkd readiness, and business traffic may trigger -it. The controller must gate those entry points too. The API does not implement -or verify that composed lifecycle. See the [API contract](sandboxd-api.md#delete-v1volumesname). +it. Gate the API and preview listeners on every node that can serve the claim. +Previously issued preview URLs can still wake it. Node-local admission does not +fence a writer on another node. Follow the [controller example](volume-controller.md) +for replication, archiving, ownership changes, and recovery. The controller +supplies the reservation and access gate. See the [API contract](sandboxd-api.md#delete-v1volumesname). ## Known limitations diff --git a/docs/volume-controller.md b/docs/volume-controller.md index e00b6a3a..2863dc79 100644 --- a/docs/volume-controller.md +++ b/docs/volume-controller.md @@ -8,7 +8,19 @@ It does not add disk lifecycle management to sandboxd. ## Prepare the claim and controller Use a `net: none` claim on its owning node. Egress-lane claims cannot hibernate. -Keep its TTL valid across the sleep interval. Use the current Python SDK with +Keep its TTL valid across the sleep interval. Set `replicate_seconds: 0` and +`archive_after_seconds: 0` on its pool, and use `on_expire: "destroy"` on the +claim. Keep that pool configured while the controller owns the claim. If the +pool is removed, the node-wide archive policy becomes the fallback. An +archived or archiving claim refuses renew with 409, so renewing alone cannot +preserve this sleep workflow when archiving is enabled. + +An existing or pending replica makes a new RW attachment return 409 volume +busy. Setting `replicate_seconds: 0` retires replicas asynchronously. Keep the +reservation and access gate closed, and retry attachment until it succeeds +after retirement. A fixed delay does not acknowledge retirement. + +Use the current Python SDK with [per-entry attach-only support](sdk-python.md#changing-volumes), Cocoon v0.7.2 or newer, and Cloud Hypervisor. @@ -45,10 +57,14 @@ Supply these application-specific operations before running the functions below: backing for this claim. The reservation must survive controller restart and the entire sleep interval. Keep it exclusively held during every step and renew any lease as needed. An assertion alone is not a lock or lease renewal. - Serialize operations for the same reservation. + Serialize operations for the same reservation. Verify the owning node too. + An owner change requires fencing the old node before another attachment. - `close_access` blocks and drains business requests before returning. It must - fail closed. Cover direct ports, guest API access, and requests that can wake - the claim. The controller's own maintenance calls may still reach silkd. + fail closed. Gate the API and preview listeners on every node that can serve + the claim, including direct ports and guest API access. A previously issued + preview URL can reach any node and wake the claim. Preview tokens have no + per-URL revocation, so stopping new URL issuance is not an access gate. + The controller's own maintenance calls may still reach silkd. - `quiesce_and_unmount` stops and reaps writers, flushes application state and filesystem writes, and ordinarily unmounts the volume. It raises if any step fails. Do not use lazy or forced unmount to turn a busy result into success. @@ -61,8 +77,12 @@ Supply these application-specific operations before running the functions below: These callbacks are not supplied by sandboxd. An in-process lock or a local boolean is not a durable reservation or an access gate for other callers. The controller must enforce their contracts even when a step fails or its -process restarts. Root-only attachment does not prevent a guest from accessing -an already attached disk, or prevent the tenant from removing it for cleanup. +process restarts. sandboxd holds no reservation for the disk once detach and +reference retirement complete. During sleep, root can attach the same catalog +entry to another claim. The controller must prevent that operation through its +own ownership policy. Root-only attachment does not prevent a guest from +accessing an already attached disk, or prevent the tenant from removing it for +cleanup. Use one attach-only writable volume for this example. The controller mounts it explicitly and owns filesystem consistency after a crash. sandboxd writes no @@ -71,50 +91,19 @@ RW entry without accounting for its [dirty-marker rules](sandboxd-api.md#attach- ## Controller functions -Copy this block into the controller and pass its operations as `VolumeHooks`. +Pass the controller operations as `VolumeHooks`. The functions do not release the reservation. Before the final `open_access` call, exceptions stop the sequence without attempting to open access or run another lifecycle operation. If a gate hook fails or has an uncertain result, that hook and the controller must recover the gate. These functions cannot prove that an external gate closed or undo a partially completed reopen. -```python -from collections.abc import Callable -from dataclasses import dataclass - -from cocoonsandbox import Sandbox - - -@dataclass(frozen=True) -class VolumeHooks: - assert_reserved: Callable[[], None] - close_access: Callable[[], None] - quiesce_and_unmount: Callable[[], None] - mount_and_verify: Callable[[], None] - open_access: Callable[[], None] - - -def hibernate_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: - hooks.close_access() - hooks.assert_reserved() - hooks.quiesce_and_unmount() - view = sb.patch_volumes(remove=[name]) - if view["volumes_want"] or view["volumes"] or view["states"] or view["converging"]: - raise RuntimeError("volume references remain; keep access closed") - sb.hibernate() - - -def resume_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: - hooks.close_access() - hooks.assert_reserved() - sb.exec("true") - volume = {"name": name, "mode": "rw", "attach_only": True} - view = sb.patch_volumes(add=[volume]) - if view["volumes_want"] != [volume] or view["volumes"] != [volume] or view["converging"]: - raise RuntimeError("volume attachment is not settled; keep access closed") - hooks.mount_and_verify() - hooks.open_access() -``` +The [tested Python example](../sdk/python/examples/volume_controller.py) defines +`VolumeHooks`, `hibernate_volume`, and `resume_volume`. Copy that module into +your controller, then supply the callbacks described above. The SDK's regular +pytest suite exercises these functions through HTTP with mocked node replies. +Those tests check request and response compatibility and failure ordering. +They do not prove guest unmount, disk integrity, or an external access gate. Call `resume_volume` for initial attachment to an empty running claim, or after sleep. It wakes through a guest command, then attaches and mounts the disk. @@ -140,6 +129,12 @@ require success before calling hibernate. Make the guest cleanup hook safe for a retry where the filesystem is already unmounted. Do not blindly rerun application writes, formatting, or an entire lifecycle sequence. +If the owning node changes, treat the claim as a new placement even if its ID +is unchanged. Keep access closed and fence the old node from the backing before +reattaching on another node. Node-local admission and catalog registration do +not provide a distributed single-writer lock. This example does not implement +failover fencing or replica transfer. + If hibernate or wake fails, inspect claim state and resume from the confirmed stage while retaining the reservation. `exec("true")` proves only silkd readiness. The claim may already have woken because another request reached diff --git a/sdk/python/examples/volume_controller.py b/sdk/python/examples/volume_controller.py new file mode 100644 index 00000000..4ffbcef7 --- /dev/null +++ b/sdk/python/examples/volume_controller.py @@ -0,0 +1,37 @@ +"""Compose volume detach and restore with controller-owned reservation and access hooks.""" + +from collections.abc import Callable +from dataclasses import dataclass + +from cocoonsandbox import Sandbox, Volume + + +@dataclass(frozen=True) +class VolumeHooks: + assert_reserved: Callable[[], None] + close_access: Callable[[], None] + quiesce_and_unmount: Callable[[], None] + mount_and_verify: Callable[[], None] + open_access: Callable[[], None] + + +def hibernate_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + hooks.quiesce_and_unmount() + view = sb.patch_volumes(remove=[name]) + if view["volumes_want"] or view["volumes"] or view["states"] or view["converging"]: + raise RuntimeError("volume references remain; keep access closed") + sb.hibernate() + + +def resume_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + sb.exec("true") + volume: Volume = {"name": name, "mode": "rw", "attach_only": True} + view = sb.patch_volumes(add=[volume]) + if view["volumes_want"] != [volume] or view["volumes"] != [volume] or view["converging"]: + raise RuntimeError("volume attachment is not settled; keep access closed") + hooks.mount_and_verify() + hooks.open_access() diff --git a/sdk/python/tests/test_volume_controller.py b/sdk/python/tests/test_volume_controller.py new file mode 100644 index 00000000..c100f3e9 --- /dev/null +++ b/sdk/python/tests/test_volume_controller.py @@ -0,0 +1,138 @@ +"""Run the documented controller functions through the SDK's HTTP boundary.""" + +import pytest +from conftest import FakeNode + +from cocoonsandbox import Client, VolumeError +from examples.volume_controller import VolumeHooks, hibernate_volume, resume_volume + +PATH = "/v1/sandboxes/sb_1" +EMPTY = {"volumes_want": [], "volumes": [], "states": [], "converging": False} +ATTACHED = { + "volumes_want": [{"name": "home", "mode": "rw"}], + "volumes": [{"name": "home", "mode": "rw"}], + "states": [{"name": "home", "state": "attached"}], + "converging": False, +} + + +@pytest.fixture +def controller(spawn_node, monkeypatch): + events = [] + replies = [] + fail = set() + gate = {"open": True} + + def hook(name): + def run(): + events.append(name) + if name == "close": + gate["open"] = False + if name in fail: + raise RuntimeError(name) + if name == "open": + gate["open"] = True + + return run + + def patch(body, path): + assert FakeNode.last_headers["Authorization"] == "Bearer root-token" + assert not gate["open"] + events.append(("patch", body)) + if replies: + return replies.pop(0) + return 200, EMPTY if "remove" in body else ATTACHED + + def hibernate(body, path): + assert FakeNode.last_headers["Authorization"] == "Bearer sandbox-token" + assert not gate["open"] + events.append("hibernate") + return 204, {} + + addr = spawn_node({("PATCH", PATH + "/volumes"): patch, ("POST", PATH + "/hibernate"): hibernate}) + client = Client(addr, api_token="root-token") + sb = client.attach(addr, "sb_1", "sandbox-token") + + def exec_true(*argv): + assert argv == ("true",) + assert not gate["open"] + events.append("wake") + return "" + + monkeypatch.setattr(sb, "exec", exec_true) + hooks = VolumeHooks(hook("reserved"), hook("close"), hook("unmount"), hook("verify"), hook("open")) + return sb, hooks, events, replies, fail, gate + + +def test_controller_cycle_uses_legacy_replies_and_opens_after_verification(controller): + sb, hooks, events, _, _, gate = controller + hibernate_volume(sb, "home", hooks) + assert not gate["open"] + resume_volume(sb, "home", hooks) + assert events == [ + "close", + "reserved", + "unmount", + ("patch", {"remove": ["home"]}), + "hibernate", + "close", + "reserved", + "wake", + ("patch", {"add": [{"name": "home", "mode": "rw", "attach_only": True}]}), + "verify", + "open", + ] + assert gate["open"] + assert sb.volumes == [{"name": "home", "mode": "rw", "attach_only": True}] + + +@pytest.mark.parametrize("failure", ["reserved", "unmount"]) +def test_controller_never_detaches_when_preconditions_fail(controller, failure): + sb, hooks, events, _, fail, gate = controller + fail.add(failure) + with pytest.raises(RuntimeError, match=failure): + hibernate_volume(sb, "home", hooks) + assert not gate["open"] + assert events == ["close", "reserved"] + (["unmount"] if failure == "unmount" else []) + + +def test_controller_failed_remove_needs_successful_retry_even_with_empty_view(controller): + sb, hooks, events, replies, _, gate = controller + replies.append((503, {**EMPTY, "error": "claim write failed"})) + with pytest.raises(VolumeError) as caught: + hibernate_volume(sb, "home", hooks) + assert caught.value.status == 503 + assert sb.volumes == sb.volumes_want == [] + assert not gate["open"] + assert "hibernate" not in events + + hibernate_volume(sb, "home", hooks) + assert [event for event in events if isinstance(event, tuple)] == [("patch", {"remove": ["home"]})] * 2 + assert events[-1] == "hibernate" + assert not gate["open"] + + +@pytest.mark.parametrize("remaining", ["volumes_want", "volumes", "states", "converging"]) +def test_controller_does_not_hibernate_an_unsettled_remove(controller, remaining): + sb, hooks, events, replies, _, gate = controller + replies.append((200, {**EMPTY, remaining: True if remaining == "converging" else [{"name": "other"}]})) + with pytest.raises(RuntimeError, match="references remain"): + hibernate_volume(sb, "home", hooks) + assert "hibernate" not in events + assert not gate["open"] + + +@pytest.mark.parametrize("failure", ["attach", "unsettled", "verify"]) +def test_controller_restore_failure_keeps_access_closed(controller, failure): + sb, hooks, events, replies, fail, gate = controller + if failure == "attach": + replies.append((400, {"error": "backing missing"})) + elif failure == "unsettled": + replies.append((200, {**ATTACHED, "converging": True})) + else: + fail.add("verify") + with pytest.raises(VolumeError if failure == "attach" else RuntimeError): + resume_volume(sb, "home", hooks) + assert not gate["open"] + assert "open" not in events + assert ("verify" in events) == (failure == "verify") From 05e29ada7f28d41c85d2b13c7ef8f4c0ad2070fd Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 05:40:34 +0000 Subject: [PATCH 45/72] pool: reproduce live writer and replica admission races --- sandboxd/pool/volume_replica_test.go | 135 +++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 sandboxd/pool/volume_replica_test.go diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go new file mode 100644 index 00000000..6812ffeb --- /dev/null +++ b/sandboxd/pool/volume_replica_test.go @@ -0,0 +1,135 @@ +package pool + +import ( + "errors" + "fmt" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestLiveWriterRefusesExistingReplica(t *testing.T) { + for _, pending := range []bool{false, true} { + for _, attachOnly := range []bool{false, true} { + for _, patch := range []bool{false, true} { + t.Run(fmt.Sprintf("pending=%t/attach_only=%t/patch=%t", pending, attachOnly, patch), func(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + if pending { + m.archives = &hookRegistry{Registry: c.reg, losePutReply: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + } else { + replicateNow(t, m, sb) + } + row, exists, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !exists || row.Consumed || !sb.HasReplica() { + t.Fatalf("replica setup failed: %+v, %t, %v", row, exists, err) + } + eng := m.eng.(*fakeEngine) + before := eng.volumeOpsLog() + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: attachOnly}} + var view types.SandboxVolumesResponse + if patch { + view, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: writer}) + } else { + view, err = m.SetVolumes(t.Context(), sb.ID, "", writer) + } + if !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("writer admitted while replica is recoverable: %v", err) + } + if len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || !slices.Equal(before, eng.volumeOpsLog()) || volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatalf("refused writer changed devices or holds: %+v", view) + } + if next, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || !ok || next.Checkpoint != row.Checkpoint || next.Consumed { + t.Fatalf("refusal changed replica: %+v, %t, %v", next, ok, getErr) + } + m.archives = c.reg + m.retireReplica(t.Context(), sb) + if sb.HasReplica() { + t.Fatal("retirement did not settle the replica") + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", writer); err != nil { + t.Fatalf("writer after replica retirement: %v", err) + } + }) + } + } + } +} + +func TestReplicaPublicationRefusesVolumeAdmittedDuringExport(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + stalled := &stallingStore{Store: m.ckpts, published: make(chan string), release: make(chan struct{})} + m.ckpts = stalled + result := make(chan error, 1) + go func() { result <- m.replicate(t.Context(), sb) }() + ck := <-stalled.published + release := func() { + select { + case <-stalled.release: + default: + close(stalled.release) + } + } + t.Cleanup(release) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + view, err := m.SetVolumes(t.Context(), sb.ID, "", writer) + if err != nil || len(view.Volumes) != 1 { + t.Fatalf("writer during checkpoint publication: %+v, %v", view, err) + } + release() + if err = <-result; !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("stale replica registered after writer admission: %v", err) + } + if _, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || ok || sb.HasReplica() || ckExists(t, m, ck) { + t.Fatalf("stale replica remains recoverable: row=%t, claim=%t, checkpoint=%t, err=%v", ok, sb.HasReplica(), ckExists(t, m, ck), getErr) + } + if got := volumeHoldersOf(m, "data"); got.writers != 1 { + t.Fatalf("refusing replica lost writer hold: %+v", got) + } + if err = m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("snapshot admission while writer is held: %v", err) + } +} + +func TestReplicaAllowsLiveReadOnlyAndCleanup(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + reader := []types.Volume{{Name: "data"}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", reader); err != nil { + t.Fatalf("reader with replica: %v", err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", reader); err != nil { + t.Fatalf("held reader retry: %v", err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("reader detach with replica: %v", err) + } +} + +func newReplicaVolumeManager(t *testing.T) (*Manager, *failoverCluster) { + t.Helper() + c := newReplicaCluster(t) + spec := archivePool(keepAnHour) + spec.ReplicateSeconds = c.replicate + cfg := &config.Config{ + APIToken: "root", DataDir: t.TempDir(), CheckpointDir: c.ckDir, + Pools: []config.PoolSpec{spec}, Mesh: &config.MeshConfig{NodeID: "a"}, + Volumes: []config.VolumeSpec{{Name: "data", Path: writeVolumeImage(t, "data", "original"), Writable: true}}, + } + backends := testBackends(t, cfg) + backends.Archives = c.reg + m, err := NewManager(t.Context(), cfg, newFakeEngine(), testSecrets(t), backends) + if err != nil { + t.Fatal(err) + } + c.renew(t, "a") + m.liftFence() + return m, c +} From 13003691af314539e7a263689cdf090e3483b70a Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 05:43:35 +0000 Subject: [PATCH 46/72] pool: exclude new writers from recoverable replicas --- sandboxd/pool/replica.go | 13 +++++--- sandboxd/pool/volume_mutation.go | 5 ++++ sandboxd/pool/volume_replica_test.go | 45 +++++++++++++++++++++++----- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 4706ddd4..4a4b015b 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -112,16 +112,21 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() - ok := m.claimed[sb.ID] == sb && sb.ArchiveCk == "" && sb.ArchivePending == "" var intent claimSnapshot - if ok { + var refusal error + switch { + case m.claimed[sb.ID] != sb || sb.ArchiveCk != "" || sb.ArchivePending != "": + refusal = errWokeMeanwhile + case m.hasVolumeHolds(sb): + refusal = ErrVolumeCapture + default: sb.ReplicaPending = ckID intent = m.store.set(sb) } m.mu.Unlock() - if !ok { + if refusal != nil { m.deleteOrphanArchiveCk(ctx, ckID) - return errWokeMeanwhile + return refusal } if err := m.store.commit(intent); err != nil { m.clearReplicaIntent(ctx, sb, ckID) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 74d61097..7ecfba4b 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -276,6 +276,11 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run sb.Transition.Unlock() return claimSnapshot{}, false, ErrVolumeCapture } + if sb.HasReplica() && slices.ContainsFunc(additions, types.Volume.RW) { + m.mu.Unlock() + sb.Transition.Unlock() + return claimSnapshot{}, false, fmt.Errorf("%w: writable attachment requires replica retirement", ErrVolumeBusy) + } if err := m.reserveResolved(run.access, resolved, sb); err != nil { m.mu.Unlock() sb.Transition.Unlock() diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index 6812ffeb..5f3a1a81 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "slices" + "sync" "testing" "github.com/cocoonstack/sandbox/sandboxd/config" @@ -69,13 +70,7 @@ func TestReplicaPublicationRefusesVolumeAdmittedDuringExport(t *testing.T) { result := make(chan error, 1) go func() { result <- m.replicate(t.Context(), sb) }() ck := <-stalled.published - release := func() { - select { - case <-stalled.release: - default: - close(stalled.release) - } - } + release := sync.OnceFunc(func() { close(stalled.release) }) t.Cleanup(release) writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} view, err := m.SetVolumes(t.Context(), sb.ID, "", writer) @@ -97,6 +92,42 @@ func TestReplicaPublicationRefusesVolumeAdmittedDuringExport(t *testing.T) { } } +func TestReplicaCaptureRefusesPendingLiveWriter(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + eng := m.eng.(*fakeEngine) + entered, resume := make(chan struct{}), make(chan struct{}) + release := sync.OnceFunc(func() { close(resume) }) + t.Cleanup(release) + eng.liveVolumeHook = func(verb string) error { + if verb == "attach" { + close(entered) + <-resume + } + return nil + } + result := make(chan error, 1) + go func() { + _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}) + result <- err + }() + <-entered + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 1 || !view.Converging { + t.Fatalf("writer did not reach pending attachment: %+v, %v", view, err) + } + if err = m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("captured pending writer: %v", err) + } + if _, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || ok || len(eng.snapSaves) != 0 { + t.Fatalf("pending writer produced snapshot or row: snapshots=%v, row=%t, err=%v", eng.snapSaves, ok, getErr) + } + release() + if err = <-result; err != nil { + t.Fatalf("pending writer did not complete: %v", err) + } +} + func TestReplicaAllowsLiveReadOnlyAndCleanup(t *testing.T) { m, _ := newReplicaVolumeManager(t) sb := mustClaim(t, m, testKey) From e89d7f67df2b8b3651068c95c71900580a71d27d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 06:08:37 +0000 Subject: [PATCH 47/72] sdk/python: cover direct constructor volume semantics --- sdk/python/tests/test_client.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 024cd9fb..b40160c1 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -7,7 +7,7 @@ import pytest from conftest import FakeNode, sandbox_at -from cocoonsandbox import APIError, Client, Template +from cocoonsandbox import APIError, Client, Sandbox, Template class KeptNode(FakeNode): @@ -583,3 +583,24 @@ def test_sandbox_handle_decodes_volume_responses(spawn_node, operation): assert sb.volumes == [{"name": "raw", "attach_only": True}, {"name": "mounted", "mount": "/data"}] assert sb.volumes_want == sb.volumes assert "attach_only" not in payload["volumes"][0] + + +@pytest.mark.parametrize( + "volume", + [ + {"name": "data"}, + {"name": "data", "mount": ""}, + {"name": "data", "attach_only": False}, + {"name": "data", "attach_only": True}, + {"name": "data", "mount": "/data"}, + ], +) +def test_sandbox_constructor_preserves_volume_input(volume): + volumes = [volume] + sb = Sandbox(Client("unused.invalid:7777"), "sb_1", "tok", "unused.invalid:7777", volumes=volumes) + assert sb.volumes == sb.volumes_want == volumes + sb.volumes[0]["name"] = "caller" + assert sb.volumes_want == volumes + sb.volumes_want[0]["name"] = "desired" + assert volumes == [volume] + assert volume["name"] == "data" From 65adba784c73eada3161c6346a17fa2e55278b3e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 06:09:11 +0000 Subject: [PATCH 48/72] sdk/python: decode wire volumes at handle creation --- sdk/python/cocoonsandbox/client.py | 4 ++-- sdk/python/cocoonsandbox/sandbox.py | 4 ++-- sdk/python/pyproject.toml | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 18f3f945..843ca135 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -18,7 +18,7 @@ from .errors import APIError, SandboxTimeout, require_field from .sandbox import Sandbox from .template import _claim_body, _template_query -from .volumes import Volume, _sandbox_response +from .volumes import Volume, _response_volumes, _sandbox_response T = TypeVar("T") @@ -181,7 +181,7 @@ def _handle_from(self, dialed: str, reply: dict[str, Any], verb: str = "claim") deadline=reply.get("deadline", ""), from_checkpoint=reply.get("from_checkpoint", ""), template_digest=reply.get("template_digest", ""), - volumes=reply.get("volumes") or [], + volumes=_response_volumes(reply.get("volumes") or []), ) def _post_json( diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 46cdd393..4981a222 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -24,7 +24,7 @@ ) from .frames import BULK_CHUNK, FS_CHUNK, KEEP_ALIVE_PROTO from .template import Template -from .volumes import SandboxVolumesResponse, Volume, _response_volumes, _volume_bodies, _volume_response +from .volumes import SandboxVolumesResponse, Volume, _volume_bodies, _volume_response if TYPE_CHECKING: from .client import Client @@ -54,7 +54,7 @@ def __init__( self.deadline = deadline self.from_checkpoint = from_checkpoint self.template_digest = template_digest - self.volumes = [cast("dict[str, str | bool]", volume) for volume in _response_volumes(volumes or [])] + self.volumes = [dict(cast("Mapping[str, str | bool]", volume)) for volume in volumes or []] self.volumes_want = [dict(volume) for volume in self.volumes] self._pool = ConnPool(client.keep_alive) self._proto = 0 diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 5a898e7d..0b87f3cd 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -13,9 +13,12 @@ readme = "README.md" [tool.setuptools.packages.find] include = ["cocoonsandbox*"] +[tool.pytest.ini_options] +pythonpath = ["."] + [tool.mypy] strict = true -files = ["cocoonsandbox"] +files = ["cocoonsandbox", "examples"] [tool.ruff] line-length = 120 From bc33984d820f7acc2a2cfdc954cb43fda92921d1 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 06:09:09 +0000 Subject: [PATCH 49/72] pool: reproduce writer starvation through replica scans --- sandboxd/pool/volume_replica_test.go | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index 5f3a1a81..1509371e 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -6,11 +6,47 @@ import ( "slices" "sync" "testing" + "time" "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/types" ) +func TestLiveWriterConvergesThroughReplicaScans(t *testing.T) { + for _, pending := range []bool{false, true} { + t.Run(fmt.Sprintf("pending=%t", pending), func(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + if pending { + m.archives = &hookRegistry{Registry: c.reg, losePutReply: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + m.archives = c.reg + } else { + replicateNow(t, m, sb) + } + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + var view types.SandboxVolumesResponse + var err error + for cycle := range 5 { + view, err = m.SetVolumes(t.Context(), sb.ID, "", writer) + if err == nil { + if len(view.Volumes) != 1 || sb.HasReplica() { + t.Fatalf("writer exposed alongside replica: %+v, replica=%t", view, sb.HasReplica()) + } + return + } + t.Logf("cycle %d: %+v, %v", cycle, view, err) + m.replicas = map[string]replicaState{sb.ID: {tried: time.Now().Add(-2 * time.Duration(replicateEvery) * time.Second)}} + m.replicateOnce(t.Context()) + waitFor(t, func() bool { return !m.replicating.Load() }) + } + t.Fatalf("writer never converged after five replica scans: %+v, %v", view, err) + }) + } +} + func TestLiveWriterRefusesExistingReplica(t *testing.T) { for _, pending := range []bool{false, true} { for _, attachOnly := range []bool{false, true} { From fa13dfb6c1961bf43059a6a492f494a790aae55e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 06:13:58 +0000 Subject: [PATCH 50/72] pool: reproduce unsafe replica retirement recovery --- sandboxd/pool/volume_replica_test.go | 57 ++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index 1509371e..53abf19d 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -47,6 +47,63 @@ func TestLiveWriterConvergesThroughReplicaScans(t *testing.T) { } } +func TestReplicaRetirementKeepsTombstoneUntilClaimCommit(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + ck := sb.ReplicaCk + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected claims write failure") } + sb.Transition.Lock() + err := m.dropReplica(t.Context(), sb) + sb.Transition.Unlock() + if err == nil { + t.Error("retirement acknowledged a failed claim commit") + } + rec, ok, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !ok || !rec.Consumed || !ckExists(t, m, ck) { + t.Fatalf("failed commit lost recovery tombstone/checkpoint: %+v, exists=%t, err=%v", rec, ok, err) + } + m.reconcileOwnRow(t.Context(), rec, time.Now()) + if rec, ok, err = c.reg.Get(t.Context(), sb.ID); err != nil || !ok || !rec.Consumed { + t.Fatalf("sweep erased unacknowledged retirement: %+v, exists=%t, err=%v", rec, ok, err) + } + restarted := c.restart(t, m) + if err = restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + restored, ok := restarted.byID(sb.ID) + if !ok { + t.Fatal("restart dropped a claim after failed retirement commit") + } + restarted.retireReplica(t.Context(), restored) + if restored.HasReplica() { + t.Fatal("restart did not finish retirement") + } +} + +func TestReplicaRecoveryDoesNotRepublishVolumeIntent(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + // Reconstruct the durable state a pending writer leaves before replica retirement. + sb.VolumesWant = []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + if err := m.store.commit(m.store.set(sb)); err != nil { + t.Fatal(err) + } + if _, err := c.reg.Consume(t.Context(), sb.ID, sb.ArchiveEpoch); err != nil { + t.Fatal(err) + } + rec, ok, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !ok || !rec.Consumed { + t.Fatalf("setup: %+v, %t, %v", rec, ok, err) + } + m.reconcileOwnRow(t.Context(), rec, time.Now()) + if rec, ok, err = c.reg.Get(t.Context(), sb.ID); err != nil || !ok || !rec.Consumed { + t.Fatalf("recovery republished desired writer: %+v, %t, %v", rec, ok, err) + } +} + func TestLiveWriterRefusesExistingReplica(t *testing.T) { for _, pending := range []bool{false, true} { for _, attachOnly := range []bool{false, true} { From 316fcf2450aba8173c5178de8cccad4ad3df3941 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 06:18:50 +0000 Subject: [PATCH 51/72] pool: retire replicas before applying reserved volume writers --- docs/sandboxd-api.md | 20 ++ docs/volume-controller.md | 12 +- sandboxd/pool/failover.go | 9 + sandboxd/pool/pool.go | 4 + sandboxd/pool/replica.go | 30 ++- sandboxd/pool/volume_mutation.go | 91 ++++--- sandboxd/pool/volume_replica_test.go | 307 +++++++++++++++++++++++- sandboxd/server/server.go | 2 + sandboxd/server/volume_mutation_test.go | 2 +- 9 files changed, 437 insertions(+), 40 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 57771b2b..0ae19a96 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -531,6 +531,26 @@ retrying. Retry the same PUT to converge, or send a new target to abandon unfinished additions. Concurrent requests serialize per sandbox without blocking renewals or new exec, file, and port connections. +A new RW target on a claim with an existing or pending replica is accepted into +`volumes_want` and reserves the volume, but returns 409 with an explicit replica +retirement error while that replica still blocks RW attachment. The normal background scan sees +this desired hold, retires the replica, and stops publishing replicas even if +`replicate_seconds` is positive. A repeated request cannot attach RW while the +replica remains unresolved. Registry failures appear in `states[].error` and +are retried by the scan. Removing the desired target cancels that reservation +after normal cleanup; release also remains available. + +After the request ends, `converging` is false unless ordinary recovery-reference +retirement is still pending. The replica scan never attaches the disk: poll the +view and retry the same PUT or PATCH to finish. A successful retry acknowledges +both replica retirement and attachment. Pending desired writers survive restart, +keep their admission holds, and prevent checkpoint publication. If the replica +clear has not been acknowledged by the claim journal, RW application also waits +for a successful snapshot commit. Since that journal writes all claims together, +an unacknowledged write for another claim can briefly delay RW application too. +A consumed replica row is retained until the local clear is acknowledged, so a +restart cannot mistake failed persistence for another node's adoption. + Before applying a target, sandboxd journals `VolumesWant` beside `Volumes`, storing it only while it differs from `Volumes`. A superseding request may first clean up an older desired-only attachment while diff --git a/docs/volume-controller.md b/docs/volume-controller.md index 2863dc79..857987ba 100644 --- a/docs/volume-controller.md +++ b/docs/volume-controller.md @@ -15,10 +15,14 @@ pool is removed, the node-wide archive policy becomes the fallback. An archived or archiving claim refuses renew with 409, so renewing alone cannot preserve this sleep workflow when archiving is enabled. -An existing or pending replica makes a new RW attachment return 409 volume -busy. Setting `replicate_seconds: 0` retires replicas asynchronously. Keep the -reservation and access gate closed, and retry attachment until it succeeds -after retirement. A fixed delay does not acknowledge retirement. +An existing or pending replica makes a new RW attachment record its desired +target and return 409 while the background scan retires the replica, even if +`replicate_seconds` is positive. Keep the reservation and access gate closed, +and retry attachment until it succeeds after retirement. No device is attached +automatically. See [the live-volume retirement contract](sandboxd-api.md#partial-failure-and-recovery). +Disabling replication remains required for this controller workflow: after +detach, the claim no longer holds a volume and could otherwise be replicated +again before wake and reattachment. A fixed delay does not acknowledge retirement. Use the current Python SDK with [per-entry attach-only support](sdk-python.md#changing-volumes), Cocoon v0.7.2 or newer, diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index dd0896f4..2b37e940 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -130,6 +130,10 @@ func (m *Manager) putRow(ctx context.Context, sb *types.Sandbox, ck string, repl return 0, nil } m.mu.Lock() + if replica && m.hasVolumeHolds(sb) { + m.mu.Unlock() + return 0, ErrVolumeCapture + } dto, deadline, held := dtoOf(sb), sb.Deadline, sb.ArchiveEpoch switch { case replica: @@ -476,6 +480,11 @@ func (m *Manager) dropStaleRow(ctx context.Context, sb *types.Sandbox, rec archi } func (m *Manager) purgeArchiveRow(ctx context.Context, rec archives.Record) { + // A failed replica clear can leave the old checkpoint in claims.json. + // Keep its consumed row until an acknowledged snapshot makes deletion safe. + if rec.Consumed && !m.store.synced() { + return + } logger := log.WithFunc("pool.purgeArchiveRow") if err := m.deleteArchiveCk(ctx, rec.Checkpoint); err != nil { logger.Warnf(ctx, "purge archive %s: delete ck %s: %v", rec.ID, rec.Checkpoint, err) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index ba77428c..f108ff35 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -102,6 +102,10 @@ var ( ErrNoEgressFork = errors.New("egress-lane sandboxes cannot fork, checkpoint, or promote: a resumed guest egresses before its fresh tap can be locked") ErrVolumeCapture = errors.New("sandboxes with volumes cannot hibernate, fork, checkpoint, or promote") ErrVolumeBusy = errors.New("volume is held by another claim") + + ErrReplicaPresent = errors.New("writable volume intent recorded; waiting for replica retirement before retry") + ErrVolumePendingCommit = errors.New("volume intent or replica retirement is not yet persisted; retry") + // Replaying a journal takes a writable mount, so readers stay out. ErrVolumeNeedsRecovery = errors.New("volume needs recovery by a writable claim") ErrArchived = errors.New("sandbox is archived; an exec or file call wakes it first") diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 4a4b015b..88b7e501 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -234,9 +234,32 @@ func (m *Manager) clearReplicaIntent(ctx context.Context, sb *types.Sandbox, ck func (m *Manager) retireReplica(ctx context.Context, sb *types.Sandbox) { sb.Transition.Lock() defer sb.Transition.Unlock() - if err := m.dropReplica(ctx, sb); err != nil && !benignSweepErr(err) { + err := m.dropReplica(ctx, sb) + if err != nil && !benignSweepErr(err) { log.WithFunc("pool.retireReplica").Warnf(ctx, "stop replicating %s: %v", sb.ID, err) } + m.mu.Lock() + defer m.mu.Unlock() + if m.claimed[sb.ID] != sb { + return + } + for _, v := range sb.VolumesWant { + if !pendingVolumeWriter(sb, v) { + continue + } + if err != nil { + if m.volumeErrors[sb] == nil { + m.volumeErrors[sb] = map[string]string{} + } + message := volumeReplicaFailed + if errors.Is(err, ErrArchiveStoreDown) { + message = volumeReplicaUnavailable + } + m.volumeErrors[sb][v.Name] = message + } else if message := m.volumeErrors[sb][v.Name]; message == volumeReplicaFailed || message == volumeReplicaUnavailable { + delete(m.volumeErrors[sb], v.Name) + } + } } // dropReplica ends sb's replication; the caller holds sb.Transition. @@ -252,12 +275,15 @@ func (m *Manager) dropReplica(ctx context.Context, sb *types.Sandbox) error { return err } var ck string - live, _ := m.commitIfLive(ctx, sb, func() { + live, err := m.commitIfLive(ctx, sb, func() { ck, sb.ReplicaCk = sb.ReplicaCk, "" }) if !live { return ErrUnknownSandbox } + if err != nil { + return fmt.Errorf("persist replica retirement: %w", err) + } m.dropTombstone(ctx, sb.ID, consumed) m.deleteOrphanArchiveCk(ctx, ck) return nil diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 7ecfba4b..0334276c 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -14,8 +14,10 @@ import ( ) const ( - volumeChangeFailed = "volume change failed; retry the desired target" - volumeChangeInterrupted = "volume change interrupted; retry the desired target" + volumeChangeFailed = "volume change failed; retry the desired target" + volumeChangeInterrupted = "volume change interrupted; retry the desired target" + volumeReplicaFailed = "replica retirement failed; background retry pending" + volumeReplicaUnavailable = "replica registry unavailable; background retry pending" ) type volumeEdit func(current, shapeContext []types.Volume) ([]types.Volume, error) @@ -130,6 +132,9 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = m.prepareVolumeTarget(ctx, sb, run, target, additions, resolved); err != nil { return view, err } + if err = m.checkVolumeApply(ctx, sb, run); err != nil { + return view, err + } m.mu.Lock() removals := volumeDifferenceByName(sb.Volumes, target) m.mu.Unlock() @@ -148,7 +153,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v m.mu.Unlock() for _, v := range pending { failedName = v.Name - if err = m.checkVolumeRun(ctx, sb, run); err != nil { + if err = m.checkVolumeApply(ctx, sb, run); err != nil { return view, err } repair, resolveErr := m.resolveCatalogVolume(ctx, v) @@ -264,31 +269,9 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru } func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) (claimSnapshot, bool, error) { - sb.Transition.Lock() - m.mu.Lock() - if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { - m.mu.Unlock() - sb.Transition.Unlock() + if err := m.admitVolumeTarget(ctx, sb, run, target, additions, resolved); err != nil { return claimSnapshot{}, false, err } - if sb.OnExpire == types.ExpireArchive && len(target) > 0 { - m.mu.Unlock() - sb.Transition.Unlock() - return claimSnapshot{}, false, ErrVolumeCapture - } - if sb.HasReplica() && slices.ContainsFunc(additions, types.Volume.RW) { - m.mu.Unlock() - sb.Transition.Unlock() - return claimSnapshot{}, false, fmt.Errorf("%w: writable attachment requires replica retirement", ErrVolumeBusy) - } - if err := m.reserveResolved(run.access, resolved, sb); err != nil { - m.mu.Unlock() - sb.Transition.Unlock() - return claimSnapshot{}, false, err - } - run.holds = append(run.holds, additions...) - m.mu.Unlock() - sb.Transition.Unlock() cleanErr := m.confirmVolumesClean(resolved) sb.Transition.Lock() defer sb.Transition.Unlock() @@ -313,6 +296,48 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run return m.store.set(sb), true, nil } +func (m *Manager) admitVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) error { + sb.Transition.Lock() + defer sb.Transition.Unlock() + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { + return err + } + if sb.OnExpire == types.ExpireArchive && len(target) > 0 { + return ErrVolumeCapture + } + if err := m.reserveResolved(run.access, resolved, sb); err != nil { + return err + } + run.holds = append(run.holds, additions...) + return nil +} + +// checkVolumeApply keeps desired writers reserved while the scan retires a replica. +// The claim snapshot must acknowledge retirement before a device can be exposed. +func (m *Manager) checkVolumeApply(ctx context.Context, sb *types.Sandbox, run *volumeRun) error { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { + return err + } + if !slices.ContainsFunc(sb.VolumesWant, func(v types.Volume) bool { return pendingVolumeWriter(sb, v) }) { + return nil + } + if sb.HasReplica() { + return ErrReplicaPresent + } + if !m.store.synced() { + return ErrVolumePendingCommit + } + return nil +} + +func pendingVolumeWriter(sb *types.Sandbox, v types.Volume) bool { + return v.RW() && volumeIndex(sb.VolumesWant, v.Name) >= 0 && volumeIndex(sb.Volumes, v.Name) < 0 +} + func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume) error { path, err := m.detachImagePath(ctx, sb, run, v.Name) if err != nil { @@ -406,6 +431,18 @@ func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { state = types.VolumePending } message := m.volumeErrors[sb][v.Name] + if pendingVolumeWriter(sb, v) { + switch { + case !m.store.synced(): + message = ErrVolumePendingCommit.Error() + case sb.HasReplica(): + if message != volumeReplicaFailed && message != volumeReplicaUnavailable { + message = ErrReplicaPresent.Error() + } + case message == volumeReplicaFailed || message == volumeReplicaUnavailable: + message = "" + } + } if message == "" && state != types.VolumeAttached && !view.Converging { message = volumeChangeInterrupted } @@ -425,7 +462,7 @@ func (m *Manager) recordVolumeError(sb *types.Sandbox, run *volumeRun, name stri return } errs := m.volumeErrors[sb] - if name != "" { + if name != "" && !errors.Is(err, ErrReplicaPresent) && !errors.Is(err, ErrVolumePendingCommit) { message := volumeChangeFailed if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { message = volumeChangeInterrupted diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index 53abf19d..9741b976 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -1,6 +1,7 @@ package pool import ( + "context" "errors" "fmt" "slices" @@ -35,14 +36,15 @@ func TestLiveWriterConvergesThroughReplicaScans(t *testing.T) { if len(view.Volumes) != 1 || sb.HasReplica() { t.Fatalf("writer exposed alongside replica: %+v, replica=%t", view, sb.HasReplica()) } - return } - t.Logf("cycle %d: %+v, %v", cycle, view, err) + t.Logf("cycle %d: wanted=%+v applied=%+v states=%+v replica=%t synced=%t error=%v", cycle, view.VolumesWant, view.Volumes, view.States, sb.HasReplica(), m.store.synced(), err) m.replicas = map[string]replicaState{sb.ID: {tried: time.Now().Add(-2 * time.Duration(replicateEvery) * time.Second)}} m.replicateOnce(t.Context()) waitFor(t, func() bool { return !m.replicating.Load() }) } - t.Fatalf("writer never converged after five replica scans: %+v, %v", view, err) + if err != nil { + t.Fatalf("writer never converged after five replica scans: %+v, %v", view, err) + } }) } } @@ -104,7 +106,282 @@ func TestReplicaRecoveryDoesNotRepublishVolumeIntent(t *testing.T) { } } -func TestLiveWriterRefusesExistingReplica(t *testing.T) { +func TestLiveWriterIntentSurvivesRestartBeforeRetirement(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { + t.Fatal(err) + } + persisted, err := newClaimStore(m.dataDir, false).load() + if err != nil || !slices.Equal(persisted[sb.ID].VolumesWant, writer) || len(persisted[sb.ID].Volumes) != 0 { + t.Fatalf("durable intent: %+v, %v", persisted, err) + } + // Consume without acknowledging it locally, then restart before the next scan. + if _, err = c.reg.Consume(t.Context(), sb.ID, sb.ArchiveEpoch); err != nil { + t.Fatal(err) + } + restarted := restartReplicaVolumeManager(t, m, c) + restored, ok := restarted.byID(sb.ID) + if !ok { + t.Fatal("pending writer lost after restart") + } + view, err := restarted.GetVolumes(sb.ID, "") + if err != nil || len(view.VolumesWant) != 1 || len(view.Volumes) != 0 || view.Converging || view.States[0].Error != ErrReplicaPresent.Error() { + t.Fatalf("restart view: %+v, %v", view, err) + } + t.Logf("restart: %+v replica=%t synced=%t", view, restored.HasReplica(), restarted.store.synced()) + if _, err = restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: writer}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("second writer after restart: %v", err) + } + rec, _, _ := c.reg.Get(t.Context(), sb.ID) + restarted.reconcileOwnRow(t.Context(), rec, time.Now()) + if rec, _, err = c.reg.Get(t.Context(), sb.ID); err != nil || !rec.Consumed { + t.Fatalf("sweep rearmed replica: %+v, %v", rec, err) + } + restarted.replicateOnce(t.Context()) + waitFor(t, func() bool { return !restarted.replicating.Load() }) + view, err = restarted.GetVolumes(sb.ID, "") + if err != nil || restored.HasReplica() || len(view.Volumes) != 0 || view.Converging { + t.Fatalf("retirement unexpectedly attached: %+v, %v", view, err) + } + t.Logf("retired: %+v replica=%t synced=%t", view, restored.HasReplica(), restarted.store.synced()) + if view, err = restarted.SetVolumes(t.Context(), sb.ID, "", writer); err != nil || len(view.Volumes) != 1 { + t.Fatalf("explicit retry: %+v, %v", view, err) + } + t.Logf("attached on retry: %+v replica=%t", view, restored.HasReplica()) +} + +func TestLiveWriterRetirementDiagnosticsSurviveRetry(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { + t.Fatal(err) + } + m.archives = &hookRegistry{Registry: c.reg, loseConsumeReply: true} + m.retireReplica(t.Context(), sb) + for range 2 { + view, err := m.SetVolumes(t.Context(), sb.ID, "", writer) + if !errors.Is(err, ErrReplicaPresent) || len(view.Volumes) != 0 || view.States[0].Error != volumeReplicaUnavailable { + t.Fatalf("retirement failure diagnostic: %+v, %v", view, err) + } + t.Logf("retire failure and retry: %+v replica=%t synced=%t", view, sb.HasReplica(), m.store.synced()) + } + m.archives = c.reg + m.retireReplica(t.Context(), sb) + if view, err := m.SetVolumes(t.Context(), sb.ID, "", writer); err != nil || len(view.Volumes) != 1 || view.States[0].Error != "" { + t.Fatalf("retry after registry recovery: %+v, %v", view, err) + } +} + +func TestLiveWriterFailedIntentDoesNotApply(t *testing.T) { + for _, cancelAfterWrite := range []bool{false, true} { + t.Run(fmt.Sprintf("cancel=%t", cancelAfterWrite), func(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + eng := m.eng.(*fakeEngine) + before := eng.volumeOpsLog() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + m.store.recommitting.Store(true) + m.store.replaceFile = func(raw []byte) error { + if cancelAfterWrite { + if err := m.store.replace(raw); err != nil { + return err + } + cancel() + return nil + } + return errors.New("injected intent write failure") + } + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + view, err := m.SetVolumes(ctx, sb.ID, "", writer) + if err == nil || len(view.VolumesWant) != 1 || len(view.Volumes) != 0 || !slices.Equal(before, eng.volumeOpsLog()) { + t.Fatalf("failed intent applied: %+v, %v", view, err) + } + t.Logf("failed/canceled intent: %+v replica=%t synced=%t error=%v", view, sb.HasReplica(), m.store.synced(), err) + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: writer}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("failure lost reservation: %v", err) + } + m.store.replaceFile = nil + if _, err = m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("remove canceled/failed target: %v", err) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("remove retained intent: %+v", got) + } + }) + } +} + +func TestLiveWriterWaitsForRetirementCommitAcknowledgment(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { + t.Fatal(err) + } + before := m.eng.(*fakeEngine).volumeOpsLog() + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected retirement persistence failure") } + m.retireReplica(t.Context(), sb) + view, err := m.GetVolumes(sb.ID, "") + if err != nil || sb.HasReplica() || m.store.synced() || len(view.States) != 1 || view.States[0].Error != ErrVolumePendingCommit.Error() { + t.Fatalf("unacknowledged clear: %+v, %v", view, err) + } + t.Logf("unacknowledged retirement: %+v replica=%t synced=%t", view, sb.HasReplica(), m.store.synced()) + if _, err = m.SetVolumes(t.Context(), sb.ID, "", writer); err == nil { + t.Fatal("failed persistence acknowledged attachment") + } + if !slices.Equal(before, m.eng.(*fakeEngine).volumeOpsLog()) { + t.Fatal("writer exposed before retirement persisted") + } + rec, _, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !rec.Consumed { + t.Fatalf("retirement tombstone: %+v, %v", rec, err) + } + m.reconcileOwnRow(t.Context(), rec, time.Now()) + if _, ok, err := c.reg.Get(t.Context(), sb.ID); err != nil || !ok { + t.Fatalf("sweep erased pending retirement: %t, %v", ok, err) + } + // Any newer whole-claim snapshot can acknowledge the clear; no special retry ticket is needed. + m.store.replaceFile = nil + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatal(err) + } + m.reconcileOwnRow(t.Context(), rec, time.Now()) + if view, err = m.SetVolumes(t.Context(), sb.ID, "", writer); err != nil || len(view.Volumes) != 1 { + t.Fatalf("retry after newer acknowledgment: %+v, %v", view, err) + } + t.Logf("newer snapshot acknowledged retirement: %+v replica=%t synced=%t", view, sb.HasReplica(), m.store.synced()) +} + +func TestLiveWriterApplyGuardRequiresAcknowledgedRetirement(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { + t.Fatal(err) + } + // Pause a request after its target commit, before its physical-apply check. + ctx, run, err := m.beginVolumeRun(t.Context(), sb, "") + if err != nil { + t.Fatal(err) + } + defer m.finishVolumeRun(sb, run) + if err = m.prepareVolumeTarget(ctx, sb, run, writer, nil, nil); err != nil { + t.Fatal(err) + } + before := m.eng.(*fakeEngine).volumeOpsLog() + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected retirement clear failure") } + m.retireReplica(t.Context(), sb) + if sb.HasReplica() || m.store.synced() { + t.Fatal("expected memory clear with unacknowledged snapshot") + } + if err = m.checkVolumeApply(ctx, sb, run); !errors.Is(err, ErrVolumePendingCommit) { + t.Fatalf("physical apply guard passed unacknowledged clear: %v", err) + } + if !slices.Equal(before, m.eng.(*fakeEngine).volumeOpsLog()) { + t.Fatal("device call before acknowledgment") + } + t.Logf("apply boundary: want=%+v applied=%+v replica=%t synced=%t error=%v", sb.VolumesWant, sb.Volumes, sb.HasReplica(), m.store.synced(), err) + m.store.replaceFile = nil + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + if err = m.checkVolumeApply(ctx, sb, run); err != nil { + t.Fatalf("acknowledged clear still blocked: %v", err) + } +} + +func TestReplicaConsumedRowPurgeWaitsForSnapshot(t *testing.T) { + for _, claimed := range []bool{false, true} { + t.Run(fmt.Sprintf("claimed=%t", claimed), func(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected clear failure") } + m.retireReplica(t.Context(), sb) + if !claimed { + m.mu.Lock() + delete(m.claimed, sb.ID) + m.store.del(sb.ID) + m.mu.Unlock() + } + rec, ok, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !ok || !rec.Consumed { + t.Fatalf("setup: %+v, %t, %v", rec, ok, err) + } + m.reconcileOwnRow(t.Context(), rec, time.Now()) + if _, ok, err = c.reg.Get(t.Context(), sb.ID); err != nil || !ok { + t.Fatalf("sweep dropped unacknowledged row: %t, %v", ok, err) + } + m.store.replaceFile = nil + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + m.reconcileOwnRow(t.Context(), rec, time.Now()) + if _, ok, err = c.reg.Get(t.Context(), sb.ID); err != nil || ok { + t.Fatalf("sweep retained acknowledged row: %t, %v", ok, err) + } + }) + } +} + +func TestLiveWriterWaitingIntentCanRelease(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { + t.Fatal(err) + } + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return volumeHoldersOf(m, "data") == (volumeHolders{}) }) + if _, ok := m.byID(sb.ID); ok { + t.Fatal("released pending claim remains") + } + if rec, ok, err := c.reg.Get(t.Context(), sb.ID); err != nil || ok && !rec.Consumed { + t.Fatalf("released replica still adoptable: %+v, %t, %v", rec, ok, err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: writer}); err != nil { + t.Fatalf("released intent retained writer admission: %v", err) + } +} + +func TestLiveWriterIntentIsDroppedWhenReplicaMoves(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}}); !errors.Is(err, ErrReplicaPresent) { + t.Fatal(err) + } + c.expire("a") + other := c.node(t, "b", t.TempDir(), keepAnHour) + if adopted, err := other.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { + t.Fatalf("adopt: %t, %v", adopted, err) + } + m.retireReplica(t.Context(), sb) + if _, ok := m.byID(sb.ID); ok { + t.Fatal("old owner kept a pending writer after replica moved") + } + moved, ok := other.byID(sb.ID) + if !ok || len(moved.VolumesWant) != 0 || len(moved.Volumes) != 0 { + t.Fatalf("old replica carried new volume intent: %+v", moved) + } +} + +func TestLiveWriterRecordsIntentBeforeRetiringReplica(t *testing.T) { for _, pending := range []bool{false, true} { for _, attachOnly := range []bool{false, true} { for _, patch := range []bool{false, true} { @@ -132,10 +409,10 @@ func TestLiveWriterRefusesExistingReplica(t *testing.T) { } else { view, err = m.SetVolumes(t.Context(), sb.ID, "", writer) } - if !errors.Is(err, ErrVolumeBusy) { + if !errors.Is(err, ErrReplicaPresent) { t.Fatalf("writer admitted while replica is recoverable: %v", err) } - if len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || !slices.Equal(before, eng.volumeOpsLog()) || volumeHoldersOf(m, "data") != (volumeHolders{}) { + if len(view.Volumes) != 0 || len(view.VolumesWant) != 1 || view.Converging || !slices.Equal(before, eng.volumeOpsLog()) || volumeHoldersOf(m, "data").writers != 1 { t.Fatalf("refused writer changed devices or holds: %+v", view) } if next, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || !ok || next.Checkpoint != row.Checkpoint || next.Consumed { @@ -257,3 +534,21 @@ func newReplicaVolumeManager(t *testing.T) (*Manager, *failoverCluster) { m.liftFence() return m, c } + +func restartReplicaVolumeManager(t *testing.T, old *Manager, c *failoverCluster) *Manager { + t.Helper() + spec := archivePool(keepAnHour) + spec.ReplicateSeconds = c.replicate + cfg := &config.Config{APIToken: "root", DataDir: old.dataDir, CheckpointDir: c.ckDir, Pools: []config.PoolSpec{spec}, Mesh: &config.MeshConfig{NodeID: old.nodeID}, Volumes: []config.VolumeSpec{{Name: "data", Path: old.catalog.entries["data"].disk.Path, Writable: true}}} + backends := testBackends(t, cfg) + backends.Archives = c.reg + m, err := NewManager(t.Context(), cfg, old.eng, testSecrets(t), backends) + if err != nil { + t.Fatal(err) + } + if err = m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + m.liftFence() + return m +} diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 257b8554..ebdd5a81 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -67,6 +67,8 @@ var poolErrHTTP = []struct { {pool.ErrNoEgressFork, http.StatusConflict, ""}, {pool.ErrVolumeCapture, http.StatusConflict, ""}, {pool.ErrVolumeBusy, http.StatusConflict, ""}, + {pool.ErrReplicaPresent, http.StatusConflict, ""}, + {pool.ErrVolumePendingCommit, http.StatusServiceUnavailable, ""}, {pool.ErrVolumeNeedsRecovery, http.StatusConflict, ""}, {pool.ErrArchived, http.StatusConflict, ""}, {pool.ErrPaused, http.StatusConflict, ""}, diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index a51dce7e..45a3dee1 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -153,7 +153,7 @@ func TestLiveVolumeRoutesHideOtherTenants(t *testing.T) { func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { const cause = "cannot unmount /data; close guest references and retry" - for _, refusal := range []error{pool.ErrVolumeBusy, pool.ErrVolumeCapture} { + for _, refusal := range []error{pool.ErrVolumeBusy, pool.ErrVolumeCapture, pool.ErrReplicaPresent} { t.Run(refusal.Error(), func(t *testing.T) { mgr := &fakeManager{setVolumes: func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) { return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "old"}}, States: []types.VolumeState{{Name: "old", State: "pending", Error: cause}}, Error: "volume change failed; retry the desired target"}, fmt.Errorf("%w: volume %q", refusal, "data") From 66939645e34b36d45a690f5253814d3758a20a14 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 06:22:11 +0000 Subject: [PATCH 52/72] pool: reproduce unpublished replica loss after failed clear --- sandboxd/pool/volume_replica_test.go | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index 9741b976..4c42007b 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -84,6 +84,46 @@ func TestReplicaRetirementKeepsTombstoneUntilClaimCommit(t *testing.T) { } } +func TestReplicaUnpublishedIntentSurvivesFailedClearAndRestart(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + m.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + pending := sb.ReplicaPending + if pending == "" || !ckExists(t, m, pending) { + t.Fatal("missing unpublished checkpoint") + } + m.archives = c.reg + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { + t.Fatal(err) + } + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected pending clear failure") } + m.retireReplica(t.Context(), sb) + persisted, err := newClaimStore(m.dataDir, false).load() + if err != nil || persisted[sb.ID].ReplicaPending != pending || len(persisted[sb.ID].VolumesWant) != 1 { + t.Fatalf("failure did not retain old disk state: %+v, %v", persisted, err) + } + if !ckExists(t, m, pending) { + t.Error("failed pending clear deleted the checkpoint used to distinguish an unpublished intent from adoption") + } + t.Logf("failed unpublished clear: memory_pending=%q durable_pending=%q checkpoint_exists=%t wanted=%+v synced=%t", sb.ReplicaPending, persisted[sb.ID].ReplicaPending, ckExists(t, m, pending), persisted[sb.ID].VolumesWant, m.store.synced()) + restarted := restartReplicaVolumeManager(t, m, c) + restored, ok := restarted.byID(sb.ID) + if !ok { + t.Fatal("restart mistook a failed pending clear for adoption and discarded the claim") + } + if len(restored.VolumesWant) != 1 || len(restored.Volumes) != 0 || volumeHoldersOf(restarted, "data").writers != 1 { + t.Fatalf("restart lost pending writer: %+v", restored) + } + if view, err := restarted.SetVolumes(t.Context(), sb.ID, "", writer); err != nil || len(view.Volumes) != 1 { + t.Fatalf("retry after restart: %+v, %v", view, err) + } +} + func TestReplicaRecoveryDoesNotRepublishVolumeIntent(t *testing.T) { m, c := newReplicaVolumeManager(t) sb := mustClaim(t, m, testKey) From dcd9e8cbb94a22527e64bb21ad65d184e3bdd29e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 06:24:56 +0000 Subject: [PATCH 53/72] pool: cover deferred replica checkpoint cleanup --- sandboxd/pool/volume_replica_test.go | 74 +++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index 4c42007b..c4d5dfae 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -124,6 +124,76 @@ func TestReplicaUnpublishedIntentSurvivesFailedClearAndRestart(t *testing.T) { } } +func TestReplicaPendingCheckpointCleanupWaitsForAcknowledgment(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + m.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + pending := sb.ReplicaPending + m.archives = c.reg + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected pending clear failure") } + m.retireReplica(t.Context(), sb) + if sb.ReplicaPending != "" || m.store.synced() { + t.Fatal("expected memory clear before snapshot acknowledgment") + } + if err := m.deleteArchiveCk(t.Context(), pending); err == nil { + t.Fatal("orphan delete ignored unacknowledged claim") + } + m.retryArchiveDeletes(t.Context()) + if !ckExists(t, m, pending) || !m.deleteMarked(pending) { + t.Fatal("cleanup discarded pending checkpoint or retry marker") + } + t.Logf("cleanup deferred: checkpoint=%s exists=%t marker=%t synced=%t", pending, ckExists(t, m, pending), m.deleteMarked(pending), m.store.synced()) + m.store.replaceFile = nil + if err := m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + m.retryArchiveDeletes(t.Context()) + if ckExists(t, m, pending) || m.deleteMarked(pending) { + t.Fatal("acknowledged cleanup leaked checkpoint or retry marker") + } + t.Logf("cleanup acknowledged: exists=%t marker=%t synced=%t", ckExists(t, m, pending), m.deleteMarked(pending), m.store.synced()) +} + +func TestReplicaSettledCheckpointCleanupRetriesAfterFailedCommit(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + previous := sb.ReplicaCk + m.archives = &hookRegistry{Registry: c.reg, losePutReply: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + pending := sb.ReplicaPending + m.archives = c.reg + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected settled intent failure") } + sb.Transition.Lock() + err := m.settleReplicaIntent(t.Context(), sb) + sb.Transition.Unlock() + if err == nil || sb.ReplicaPending != "" || sb.ReplicaCk != pending || m.store.synced() { + t.Fatalf("settle state: %+v, %v", sb, err) + } + if !ckExists(t, m, previous) || !m.deleteMarked(previous) { + t.Fatal("failed settle lost old checkpoint or its queued cleanup") + } + m.retryArchiveDeletes(t.Context()) + if !ckExists(t, m, previous) { + t.Fatal("old checkpoint deleted before clear acknowledgment") + } + m.store.replaceFile = nil + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + m.retryArchiveDeletes(t.Context()) + if ckExists(t, m, previous) || m.deleteMarked(previous) || !ckExists(t, m, pending) { + t.Fatal("acknowledged settle failed to clean old checkpoint or removed current checkpoint") + } +} + func TestReplicaRecoveryDoesNotRepublishVolumeIntent(t *testing.T) { m, c := newReplicaVolumeManager(t) sb := mustClaim(t, m, testKey) @@ -286,8 +356,8 @@ func TestLiveWriterWaitsForRetirementCommitAcknowledgment(t *testing.T) { t.Fatalf("retirement tombstone: %+v, %v", rec, err) } m.reconcileOwnRow(t.Context(), rec, time.Now()) - if _, ok, err := c.reg.Get(t.Context(), sb.ID); err != nil || !ok { - t.Fatalf("sweep erased pending retirement: %t, %v", ok, err) + if _, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || !ok { + t.Fatalf("sweep erased pending retirement: %t, %v", ok, getErr) } // Any newer whole-claim snapshot can acknowledge the clear; no special retry ticket is needed. m.store.replaceFile = nil From 789fbe233db90abd7bbf12b2e35054b9b58d194f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 06:25:39 +0000 Subject: [PATCH 54/72] pool: retain replica checkpoints until claim clears persist --- docs/sandboxd-api.md | 3 +++ sandboxd/pool/archive.go | 11 +++++++++-- sandboxd/pool/replica.go | 33 +++++++++++++++++++++------------ 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 0ae19a96..90ff1f88 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -550,6 +550,9 @@ for a successful snapshot commit. Since that journal writes all claims together, an unacknowledged write for another claim can briefly delay RW application too. A consumed replica row is retained until the local clear is acknowledged, so a restart cannot mistake failed persistence for another node's adoption. +Unpublished replica checkpoints and their queued cleanup markers follow the same +acknowledgment boundary. Archive checkpoint cleanup pauses while any claim write +is unacknowledged and resumes after a successful snapshot commit. Before applying a target, sandboxd journals `VolumesWant` beside `Volumes`, storing it only while it differs from `Volumes`. A diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index af26aa17..f401960d 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -355,7 +355,14 @@ func (m *Manager) deleteArchiveCk(ctx context.Context, ckID string) error { if err := m.markArchiveCk(ckID); err != nil { return fmt.Errorf("track: %w", err) } - if err := m.deleteCkLocked(ctx, ckID); err != nil { + unlock := m.lockRec(ckID) + if !m.store.synced() { + unlock() + return errors.New("checkpoint cleanup waits for claim persistence") + } + err := m.ckpts.Delete(ctx, ckID) + unlock() + if err != nil { return err } if err := m.clearArchiveCk(ckID); err != nil { @@ -386,7 +393,7 @@ func (m *Manager) retryArchiveDeletes(ctx context.Context) { func (m *Manager) retryArchiveDelete(ctx context.Context, ckID string) { unlock := m.lockRec(ckID) - if m.archiveCkPinned(ckID) || !m.deleteMarked(ckID) { + if !m.store.synced() || m.archiveCkPinned(ckID) || !m.deleteMarked(ckID) { unlock() return } diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 88b7e501..b8e04f23 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -129,19 +129,17 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { return refusal } if err := m.store.commit(intent); err != nil { - m.clearReplicaIntent(ctx, sb, ckID) - m.deleteOrphanArchiveCk(ctx, ckID) - return fmt.Errorf("replicate %s: persist intent: %w", sb.ID, err) + clearErr := m.clearReplicaIntent(ctx, sb, ckID) + return fmt.Errorf("replicate %s: persist intent: %w", sb.ID, errors.Join(err, clearErr)) } epoch, err := m.replaceReplica(ctx, sb, ckID) switch { case errors.Is(err, archives.ErrLost): - m.clearReplicaIntent(ctx, sb, ckID) - m.deleteOrphanArchiveCk(ctx, ckID) + clearErr := m.clearReplicaIntent(ctx, sb, ckID) if settleErr := m.refusedReplace(ctx, sb); settleErr != nil && !errors.Is(settleErr, ErrUnknownSandbox) { return fmt.Errorf("replicate %s: settle refused write: %w", sb.ID, settleErr) } - return fmt.Errorf("replicate %s: registry: %w", sb.ID, err) + return fmt.Errorf("replicate %s: registry: %w", sb.ID, errors.Join(err, clearErr)) case err != nil: return fmt.Errorf("replicate %s: registry: %w", sb.ID, err) } @@ -189,7 +187,7 @@ func (m *Manager) settleReplicaIntent(ctx context.Context, sb *types.Sandbox) er return ErrUnknownSandbox case ok && !rec.Consumed && rec.Checkpoint == pending: var prev string - live, _ := m.commitIfLive(ctx, sb, func() { + live, commitErr := m.commitIfLive(ctx, sb, func() { prev, sb.ReplicaCk, sb.ReplicaPending, sb.ArchiveEpoch = sb.ReplicaCk, pending, "", rec.Epoch }) if !live { @@ -198,11 +196,12 @@ func (m *Manager) settleReplicaIntent(ctx context.Context, sb *types.Sandbox) er if prev != "" { m.deleteOrphanArchiveCk(ctx, prev) } + if commitErr != nil { + return fmt.Errorf("persist settled replica: %w", commitErr) + } return nil } - m.clearReplicaIntent(ctx, sb, pending) - m.deleteOrphanArchiveCk(ctx, pending) - return nil + return m.clearReplicaIntent(ctx, sb, pending) } // replicaGone reports whether the absence of sb's row means another node adopted the claim and finished with it. @@ -223,12 +222,22 @@ func (m *Manager) replicaGone(ctx context.Context, sb *types.Sandbox) (bool, err return false, err } -func (m *Manager) clearReplicaIntent(ctx context.Context, sb *types.Sandbox, ck string) { - _, _ = m.commitIfLive(ctx, sb, func() { +func (m *Manager) clearReplicaIntent(ctx context.Context, sb *types.Sandbox, ck string) error { + live, err := m.commitIfLive(ctx, sb, func() { if sb.ReplicaPending == ck { sb.ReplicaPending = "" } }) + // Queue cleanup even on failure, but the delete boundary must wait for the + // full claim snapshot so restart can still identify an unpublished intent. + m.deleteOrphanArchiveCk(ctx, ck) + if !live { + return ErrUnknownSandbox + } + if err != nil { + return fmt.Errorf("persist replica intent clear: %w", err) + } + return nil } func (m *Manager) retireReplica(ctx context.Context, sb *types.Sandbox) { From a625c99ba02955940dff0c2e47478ecef5140729 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 06:57:58 +0000 Subject: [PATCH 55/72] pool: reproduce unrelated writes fencing volume operations --- .../pool/volume_replication_policy_test.go | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 sandboxd/pool/volume_replication_policy_test.go diff --git a/sandboxd/pool/volume_replication_policy_test.go b/sandboxd/pool/volume_replication_policy_test.go new file mode 100644 index 00000000..f6585295 --- /dev/null +++ b/sandboxd/pool/volume_replication_policy_test.go @@ -0,0 +1,107 @@ +package pool + +import ( + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/store" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestLiveWriterDoesNotWaitForAnotherClaimsRenewal(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + other := mustClaim(t, m, testKey) + attached := make(chan struct{}, 1) + eng.liveVolumeHook = func(op string) error { + if op == "attach" { + attached <- struct{}{} + } + return nil + } + resume := make(chan struct{}) + var once sync.Once + release := func() { once.Do(func() { close(resume) }) } + defer release() + renewed := make(chan error, 1) + var writes atomic.Int32 + m.store.replaceFile = func(raw []byte) error { + switch writes.Add(1) { + case 1: + if err := m.store.replace(raw); err != nil { + return err + } + before := m.store.mark().seq + go func() { + _, err := m.Renew(t.Context(), other.ID, Cred{Token: other.Token}, time.Hour, "") + renewed <- err + }() + deadline := time.Now().Add(3 * time.Second) + for m.store.mark().seq == before { + if time.Now().After(deadline) { + return errors.New("renew did not stage its unrelated claim") + } + time.Sleep(time.Millisecond) + } + return nil + case 2: + <-resume + } + return m.store.replace(raw) + } + changed := make(chan error, 1) + go func() { + _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}}) + changed <- err + }() + finished := false + select { + case <-attached: + case err := <-changed: + finished = true + t.Errorf("unrelated renewal rejected writable attach: %v", err) + case <-time.After(3 * time.Second): + t.Error("unrelated renewal prevented writable device application") + } + release() + if !finished { + if err := <-changed; err != nil { + t.Errorf("writable attach after renewal: %v", err) + } + } + if err := <-renewed; err != nil { + t.Fatal(err) + } + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.Volumes) != 1 { + t.Fatalf("writer did not attach: %+v, %v", view, err) + } +} + +func TestArchiveCheckpointCleanupIgnoresUnrelatedClaimWrites(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + checkpoint, _, err := m.publishCheckpoint(t.Context(), sb, store.CheckpointID(randHex(8)), "orphan", "", true) + if err != nil { + t.Fatal(err) + } + ck := checkpoint.ID + other := mustClaim(t, m, testKey) + m.mu.Lock() + other.LeaseSeconds++ + m.store.set(other) + m.mu.Unlock() + if m.store.synced() { + t.Fatal("expected an unrelated unacknowledged claim write") + } + if err := m.deleteArchiveCk(t.Context(), ck); err != nil { + t.Fatalf("unrelated claim delayed checkpoint cleanup: %v", err) + } + if ckExists(t, m, ck) || m.deleteMarked(ck) { + t.Fatal("unreferenced checkpoint or delete marker remained") + } +} From 86540c8b371b1697d0b445929bdb4de0476c137d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 07:16:35 +0000 Subject: [PATCH 56/72] pool: require non-replicating pools for live writable volumes --- docs/sandboxd-api.md | 54 +-- docs/volume-controller.md | 20 +- sandboxd/pool/archive.go | 10 +- sandboxd/pool/archive_test.go | 13 +- sandboxd/pool/checkpoint.go | 19 +- sandboxd/pool/claim.go | 5 +- sandboxd/pool/claims.go | 106 ++++-- sandboxd/pool/claims_test.go | 116 +++++++ sandboxd/pool/failover.go | 13 +- sandboxd/pool/pool.go | 4 +- sandboxd/pool/reconcile.go | 5 +- sandboxd/pool/replica.go | 25 +- sandboxd/pool/volume_mutation.go | 68 ++-- sandboxd/pool/volume_replica_test.go | 309 ++---------------- .../pool/volume_replication_policy_test.go | 193 +++++++++++ sandboxd/pool/volume_retirement_test.go | 2 +- sandboxd/server/server.go | 2 +- sandboxd/server/volume_mutation_test.go | 2 +- 18 files changed, 537 insertions(+), 429 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 90ff1f88..6f8949cb 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -531,28 +531,38 @@ retrying. Retry the same PUT to converge, or send a new target to abandon unfinished additions. Concurrent requests serialize per sandbox without blocking renewals or new exec, file, and port connections. -A new RW target on a claim with an existing or pending replica is accepted into -`volumes_want` and reserves the volume, but returns 409 with an explicit replica -retirement error while that replica still blocks RW attachment. The normal background scan sees -this desired hold, retires the replica, and stops publishing replicas even if -`replicate_seconds` is positive. A repeated request cannot attach RW while the -replica remains unresolved. Registry failures appear in `states[].error` and -are retried by the scan. Removing the desired target cancels that reservation -after normal cleanup; release also remains available. - -After the request ends, `converging` is false unless ordinary recovery-reference -retirement is still pending. The replica scan never attaches the disk: poll the -view and retry the same PUT or PATCH to finish. A successful retry acknowledges -both replica retirement and attachment. Pending desired writers survive restart, -keep their admission holds, and prevent checkpoint publication. If the replica -clear has not been acknowledged by the claim journal, RW application also waits -for a successful snapshot commit. Since that journal writes all claims together, -an unacknowledged write for another claim can briefly delay RW application too. -A consumed replica row is retained until the local clear is acknowledged, so a -restart cannot mistake failed persistence for another node's adoption. -Unpublished replica checkpoints and their queued cleanup markers follow the same -acknowledgment boundary. Archive checkpoint cleanup pauses while any claim write -is unacknowledged and resumes after a successful snapshot commit. +Adding a live RW volume requires a pool with `replicate_seconds: 0`. A +replicating pool returns 409 `live writable volumes require a non-replicating +pool` before recording the new target, reserving the volume, or changing devices. +This restriction applies to live additions, including attach-only RW. An RW +volume attached in the initial claim remains supported, and existing RW volumes +can be retained or removed while replication is configured. RO live changes +remain available. The effective pool is the claim's policy source when it has +one. Changing `replicate_seconds` requires a daemon restart. + +A non-replicating pool can still have an existing or pending replica after a +restart. Live RW additions return 409 until that claim's replicas and recovery +checkpoint references have retired. The background scan retires existing +replicas; keep the controller's reservation and access gate closed, then retry +the request. A failed claim write retains the affected checkpoint reference +until a successful snapshot acknowledges its removal. Unrelated claims' writes +and checkpoint references do not refuse this attachment. + +Older versions may have already recorded a desired RW target while replication +was enabled. These targets remain visible after restart and retain their holds. +They cannot attach while the pool replicates, but callers can remove them or +release the claim. Cleanup of applied volumes runs before a remaining pending RW +is refused, so a remove request can complete an RO detach and still return a +partial 409 for that pending writer. Inspect the returned view before retrying. +After a request ends, `converging` is false unless ordinary recovery-reference +retirement remains active; no background worker attaches the desired disk. + +Consumed replica rows, unpublished checkpoints and queued deletion markers are +retained while the affected checkpoint remains in a current claim projection, +an acknowledged snapshot, or an attempted replacement whose result is uncertain. +A later acknowledged snapshot retires removed references and permits cleanup. +Checkpoint deletion waits only for actual references to that checkpoint, not +for a node-wide pause in claim writes. Before applying a target, sandboxd journals `VolumesWant` beside `Volumes`, storing it only while it differs from `Volumes`. A diff --git a/docs/volume-controller.md b/docs/volume-controller.md index 857987ba..cf0633ad 100644 --- a/docs/volume-controller.md +++ b/docs/volume-controller.md @@ -15,14 +15,18 @@ pool is removed, the node-wide archive policy becomes the fallback. An archived or archiving claim refuses renew with 409, so renewing alone cannot preserve this sleep workflow when archiving is enabled. -An existing or pending replica makes a new RW attachment record its desired -target and return 409 while the background scan retires the replica, even if -`replicate_seconds` is positive. Keep the reservation and access gate closed, -and retry attachment until it succeeds after retirement. No device is attached -automatically. See [the live-volume retirement contract](sandboxd-api.md#partial-failure-and-recovery). -Disabling replication remains required for this controller workflow: after -detach, the claim no longer holds a volume and could otherwise be replicated -again before wake and reattachment. A fixed delay does not acknowledge retirement. +Live RW additions require `replicate_seconds: 0`; a replicating pool returns +409 without recording the new target or reserving the volume. Changing this +setting requires a daemon restart. A non-replicating pool may still hold an old +replica or recovery checkpoint reference. Keep the reservation and access gate +closed and retry attachment after that claim's retirement has been acknowledged. +A fixed delay does not acknowledge retirement. See +[the live-volume retirement contract](sandboxd-api.md#partial-failure-and-recovery). + +Keep replication disabled throughout this controller workflow. After detach, +the claim no longer holds a volume and could otherwise be replicated again +before wake and reattachment. RW volumes attached in the initial claim remain +supported on replicating pools, but this does not permit later live RW additions. Use the current Python SDK with [per-entry attach-only support](sdk-python.md#changing-volumes), Cocoon v0.7.2 or newer, diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index f401960d..3a85a3d0 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -18,6 +18,8 @@ import ( const archiveDeleteDir = "archive-deletes" +var errCheckpointReferenced = errors.New("checkpoint is referenced by a claim") + // archive*For resolve a key's thresholds (pool's, else node default); callers hold m.mu. func (m *Manager) archiveAfterFor(key types.PoolKey) time.Duration { if p, ok := m.activePool(key); ok { @@ -280,7 +282,7 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc } func (m *Manager) deleteOrphanArchiveCk(ctx context.Context, ckID string) { - if err := m.deleteArchiveCk(ctx, ckID); err != nil { + if err := m.deleteArchiveCk(ctx, ckID); err != nil && !errors.Is(err, errCheckpointReferenced) { log.WithFunc("pool.deleteOrphanArchiveCk").Warnf(ctx, "delete orphaned archive ck %s: %v", ckID, err) } } @@ -356,9 +358,9 @@ func (m *Manager) deleteArchiveCk(ctx context.Context, ckID string) error { return fmt.Errorf("track: %w", err) } unlock := m.lockRec(ckID) - if !m.store.synced() { + if _, referenced := m.store.checkpointRefs()[ckID]; referenced { unlock() - return errors.New("checkpoint cleanup waits for claim persistence") + return errCheckpointReferenced } err := m.ckpts.Delete(ctx, ckID) unlock() @@ -393,7 +395,7 @@ func (m *Manager) retryArchiveDeletes(ctx context.Context) { func (m *Manager) retryArchiveDelete(ctx context.Context, ckID string) { unlock := m.lockRec(ckID) - if !m.store.synced() || m.archiveCkPinned(ckID) || !m.deleteMarked(ckID) { + if m.archiveCkPinned(ckID) || !m.deleteMarked(ckID) { unlock() return } diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index c9ffeee9..468724f9 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -680,11 +680,14 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { m.retryArchiveDeletes(t.Context()) close(retried) }() - waitFor(t, func() bool { - m.recLocksMu.Lock() - defer m.recLocksMu.Unlock() - return m.recRefs[ck] >= 2 - }) + select { + case <-retried: + case <-time.After(3 * time.Second): + t.Fatal("retry did not defer the checkpoint still referenced by the journal") + } + if !ckExists(t, m, ck) { + t.Fatal("retry deleted the checkpoint before wake persistence completed") + } breakStore(t, m) m.store.writeMu.Unlock() locked = false diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index e02b70b2..95eb05cd 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -5,7 +5,6 @@ import ( "encoding/json/v2" "errors" "fmt" - "maps" "os" "path/filepath" "slices" @@ -318,16 +317,16 @@ func (m *Manager) vetoIfHealPending(ckptID string) { func (m *Manager) pinnedArchiveCks() map[string]struct{} { m.mu.Lock() defer m.mu.Unlock() - pinned := maps.Clone(m.pendingCks) + pinned := m.store.checkpointRefs() + if pinned == nil { + pinned = map[string]struct{}{} + } + for id := range m.pendingCks { + pinned[id] = struct{}{} + } for _, sb := range m.claimed { - if sb.ArchiveCk != "" { - pinned[sb.ArchiveCk] = struct{}{} - } - if sb.ReplicaCk != "" { - pinned[sb.ReplicaCk] = struct{}{} - } - if sb.ReplicaPending != "" { - pinned[sb.ReplicaPending] = struct{}{} + for _, id := range checkpointIDs(sb) { + pinned[id] = struct{}{} } } return pinned diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 6b20f440..eb3f693e 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -4,6 +4,7 @@ import ( "cmp" "context" "crypto/subtle" + "errors" "fmt" "net" "slices" @@ -696,10 +697,8 @@ func (m *Manager) pauseExpired(ctx context.Context, sb *types.Sandbox) error { func (m *Manager) purgeArchiveCk(ctx context.Context, id, ck, tenant string) { logger := log.WithFunc("pool.purgeArchiveCk") - if err := m.deleteCkLocked(ctx, ck); err != nil { + if err := m.deleteArchiveCk(ctx, ck); err != nil && !errors.Is(err, errCheckpointReferenced) { logger.Warnf(ctx, "delete archive ck %s: %v", ck, err) - } else if err := m.clearArchiveCk(ck); err != nil { - logger.Warnf(ctx, "clear archive ck %s: %v", ck, err) } m.counters.archiveDeletes.Add(1) m.recordUsage(ctx, usageEvent{Event: "archive_delete", ID: id, Reference: ck, Tenant: tenant}) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 768e63cb..144a0a42 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -69,11 +69,18 @@ type claimDTO struct { // claimRow keeps its encoded member until set replaces the row; set never edits one. type claimRow struct { id string - refs []types.Volume + refs claimRefs dto claimDTO raw []byte } +type claimRefs struct { + volumes []types.Volume + checkpoints []string +} + +func (r claimRefs) empty() bool { return len(r.volumes) == 0 && len(r.checkpoints) == 0 } + // claimStore persists claimed sandboxes across daemon restarts; warm VMs are not persisted. type claimStore struct { path string @@ -87,8 +94,8 @@ type claimStore struct { rows map[string]*claimRow seq uint64 written uint64 - ackRefs map[string][]types.Volume - attemptRefs map[string][]types.Volume + ackRefs map[string]claimRefs + attemptRefs map[string]claimRefs replaceFile func([]byte) error recommitting atomic.Bool @@ -126,9 +133,9 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { s.mu.Lock() s.ackRefs = nil for _, sb := range claims { - if row := newClaimRow(sb); len(row.refs) > 0 { + if row := newClaimRow(sb); !row.refs.empty() { if s.ackRefs == nil { - s.ackRefs = map[string][]types.Volume{} + s.ackRefs = map[string]claimRefs{} } s.ackRefs[row.id] = row.refs } @@ -194,16 +201,19 @@ func (s *claimStore) commit(snap claimSnapshot) error { return nil } rows, seq := slices.AppendSeq(make([]*claimRow, 0, len(s.rows)), maps.Values(s.rows)), s.seq - s.mu.Unlock() - s.buf = append(s.buf[:0], '{') - var refs map[string][]types.Volume - for i, row := range rows { - if len(row.refs) > 0 { + var refs map[string]claimRefs + for _, row := range rows { + if !row.refs.empty() { if refs == nil { - refs = map[string][]types.Volume{} + refs = map[string]claimRefs{} } refs[row.id] = row.refs } + } + s.attemptRefs = mergeRecoveryRefs(s.attemptRefs, refs) + s.mu.Unlock() + s.buf = append(s.buf[:0], '{') + for i, row := range rows { if row.raw == nil { raw, err := encodeRow(&row.dto) if err != nil { @@ -217,9 +227,6 @@ func (s *claimStore) commit(snap claimSnapshot) error { s.buf = append(s.buf, row.raw...) } s.buf = append(s.buf, '}') - s.mu.Lock() - s.attemptRefs = mergeRecoveryRefs(s.attemptRefs, refs) - s.mu.Unlock() replace := s.replace if s.replaceFile != nil { replace = s.replaceFile @@ -326,7 +333,17 @@ func newClaimRow(sb *types.Sandbox) *claimRow { refs = append(refs, v) } } - return &claimRow{id: sb.ID, refs: refs, dto: dtoOf(sb)} + return &claimRow{id: sb.ID, refs: claimRefs{volumes: refs, checkpoints: checkpointIDs(sb)}, dto: dtoOf(sb)} +} + +func checkpointIDs(sb *types.Sandbox) []string { + var checkpoints []string + for _, id := range []string{sb.ArchiveCk, sb.ArchivePending, sb.ReplicaCk, sb.ReplicaPending} { + if id != "" && !slices.Contains(checkpoints, id) { + checkpoints = append(checkpoints, id) + } + } + return checkpoints } func (s *claimStore) recoveryVolumes(owner ...string) map[string][]types.Volume { @@ -334,21 +351,70 @@ func (s *claimStore) recoveryVolumes(owner ...string) map[string][]types.Volume defer s.mu.Unlock() if len(owner) > 0 { id := owner[0] - refs := appendRecoveryRefs(slices.Clone(s.ackRefs[id]), s.attemptRefs[id]) + refs := appendRecoveryRefs(slices.Clone(s.ackRefs[id].volumes), s.attemptRefs[id].volumes) if len(refs) == 0 { return nil } return map[string][]types.Volume{id: refs} } - return mergeRecoveryRefs(mergeRecoveryRefs(nil, s.ackRefs), s.attemptRefs) + var volumes map[string][]types.Volume + for id, refs := range mergeRecoveryRefs(mergeRecoveryRefs(nil, s.ackRefs), s.attemptRefs) { + if len(refs.volumes) > 0 { + if volumes == nil { + volumes = map[string][]types.Volume{} + } + volumes[id] = refs.volumes + } + } + return volumes } -func mergeRecoveryRefs(dst, src map[string][]types.Volume) map[string][]types.Volume { +func (s *claimStore) checkpointRefs(owner ...string) map[string]struct{} { + s.mu.Lock() + defer s.mu.Unlock() + var refs map[string]struct{} + add := func(row claimRefs) { + for _, ck := range row.checkpoints { + if refs == nil { + refs = map[string]struct{}{} + } + refs[ck] = struct{}{} + } + } + if len(owner) > 0 { + id := owner[0] + if row := s.rows[id]; row != nil { + add(row.refs) + } + add(s.ackRefs[id]) + add(s.attemptRefs[id]) + return refs + } + for _, row := range s.rows { + add(row.refs) + } + for _, row := range s.ackRefs { + add(row) + } + for _, row := range s.attemptRefs { + add(row) + } + return refs +} + +func mergeRecoveryRefs(dst, src map[string]claimRefs) map[string]claimRefs { if dst == nil && len(src) > 0 { - dst = make(map[string][]types.Volume, len(src)) + dst = make(map[string]claimRefs, len(src)) } for id, refs := range src { - dst[id] = appendRecoveryRefs(dst[id], refs) + row := dst[id] + row.volumes = appendRecoveryRefs(row.volumes, refs.volumes) + for _, ck := range refs.checkpoints { + if !slices.Contains(row.checkpoints, ck) { + row.checkpoints = append(row.checkpoints, ck) + } + } + dst[id] = row } return dst } diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index f40870b0..8ddf42d5 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -1,6 +1,7 @@ package pool import ( + "encoding/json/v2" "errors" "io" "io/fs" @@ -18,6 +19,121 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +func TestClaimCheckpointReferencesTrackPossibleSnapshots(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", ArchiveCk: "archive", ArchivePending: "archive-pending", ReplicaCk: "replica", ReplicaPending: "replica-pending", FromCheckpoint: "branch-origin"} + snap := s.set(sb) + if refs := s.checkpointRefs(sb.ID); len(refs) != 4 { + t.Fatalf("current references: %+v", refs) + } + if err := s.commit(snap); err != nil { + t.Fatal(err) + } + sb.ReplicaPending = "uncertain-replica" + s.replaceFile = func(raw []byte) error { + if err := s.replace(raw); err != nil { + return err + } + return errors.New("injected lost replacement acknowledgement") + } + if err := s.commit(s.set(sb)); err == nil { + t.Fatal("replacement unexpectedly acknowledged") + } + loaded, err := newClaimStore(filepath.Dir(s.path), false).load() + if err != nil || loaded[sb.ID].ReplicaPending != "uncertain-replica" { + t.Fatalf("lost acknowledgement fixture: %+v, %v", loaded, err) + } + sb.ArchiveCk, sb.ArchivePending, sb.ReplicaCk, sb.ReplicaPending = "", "", "", "" + s.set(sb) + s.set(&types.Sandbox{ID: "other", ReplicaPending: "other-replica"}) + refs := s.checkpointRefs(sb.ID) + if len(refs) != 5 { + t.Fatalf("lost snapshot references: %+v", refs) + } + delete(refs, "uncertain-replica") + if _, retained := s.checkpointRefs(sb.ID)["uncertain-replica"]; !retained { + t.Fatal("caller mutated the retained reference set") + } + if len(s.checkpointRefs("unknown")) != 0 || len(s.checkpointRefs("other")) != 1 { + t.Fatal("checkpoint references crossed claim ownership") + } + s.replaceFile = nil + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if len(s.checkpointRefs(sb.ID)) != 0 || len(s.checkpointRefs()) != 1 { + t.Fatal("acknowledged snapshot did not retire only the removed references") + } +} + +func TestClaimCapturedCheckpointReferenceSurvivesConcurrentClear(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", ReplicaPending: "captured-replica"} + entered, resume := make(chan struct{}), make(chan struct{}) + release := sync.OnceFunc(func() { close(resume) }) + defer release() + s.replaceFile = func(raw []byte) error { + close(entered) + <-resume + var rows map[string]*types.Sandbox + if err := json.Unmarshal(raw, &rows); err != nil { + return err + } + if rows[sb.ID].ReplicaPending != "captured-replica" { + return errors.New("captured write lost its original checkpoint") + } + return s.replace(raw) + } + snap := s.set(sb) + committed := make(chan error, 1) + go func() { committed <- s.commit(snap) }() + <-entered + sb.ReplicaPending = "" + s.set(sb) + if _, retained := s.checkpointRefs(sb.ID)["captured-replica"]; !retained { + t.Error("captured replacement reference disappeared before acknowledgement") + } + release() + if err := <-committed; err != nil { + t.Fatal(err) + } + s.replaceFile = nil + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if len(s.checkpointRefs(sb.ID)) != 0 { + t.Fatal("later acknowledgement retained an obsolete checkpoint") + } +} + +func TestClaimCheckpointReferencesSurviveFailedStartupRewrite(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", ReplicaPending: "before-recovery"} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + s = newClaimStore(filepath.Dir(s.path), false) + claims, err := s.load() + if err != nil { + t.Fatal(err) + } + claims[sb.ID].ReplicaPending = "" + s.replaceFile = func([]byte) error { return errors.New("startup rewrite failed") } + if err = s.save(claims); err == nil { + t.Fatal("startup rewrite unexpectedly acknowledged") + } + if _, retained := s.checkpointRefs(sb.ID)["before-recovery"]; !retained { + t.Fatal("failed startup rewrite retired the loaded checkpoint") + } + s.replaceFile = nil + if err = s.save(claims); err != nil { + t.Fatal(err) + } + if len(s.checkpointRefs(sb.ID)) != 0 { + t.Fatal("successful startup rewrite retained the old checkpoint") + } +} + func TestRecommitKeepsOneRetrierWhileWritesFail(t *testing.T) { m := newTestManager(t, newFakeEngine()) sb := mustClaim(t, m, testKey) diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 2b37e940..f0137626 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -303,7 +303,9 @@ func (m *Manager) reputArchive(ctx context.Context, sb *types.Sandbox, held int6 epoch, err = m.putReplica(ctx, sb, sb.ReplicaCk) } if err != nil { - log.WithFunc("pool.reputArchive").Warnf(ctx, "re-put archive row %s: %v", sb.ID, err) + if !errors.Is(err, ErrVolumeCapture) { + log.WithFunc("pool.reputArchive").Warnf(ctx, "re-put archive row %s: %v", sb.ID, err) + } return } _, _ = m.commitIfLive(ctx, sb, func() { sb.ArchiveEpoch = epoch }) @@ -480,14 +482,11 @@ func (m *Manager) dropStaleRow(ctx context.Context, sb *types.Sandbox, rec archi } func (m *Manager) purgeArchiveRow(ctx context.Context, rec archives.Record) { - // A failed replica clear can leave the old checkpoint in claims.json. - // Keep its consumed row until an acknowledged snapshot makes deletion safe. - if rec.Consumed && !m.store.synced() { - return - } logger := log.WithFunc("pool.purgeArchiveRow") if err := m.deleteArchiveCk(ctx, rec.Checkpoint); err != nil { - logger.Warnf(ctx, "purge archive %s: delete ck %s: %v", rec.ID, rec.Checkpoint, err) + if !errors.Is(err, errCheckpointReferenced) { + logger.Warnf(ctx, "purge archive %s: delete ck %s: %v", rec.ID, rec.Checkpoint, err) + } return } switch err := m.archives.Delete(ctx, rec.ID, rec.Epoch); { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index f108ff35..baaaa864 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -103,8 +103,8 @@ var ( ErrVolumeCapture = errors.New("sandboxes with volumes cannot hibernate, fork, checkpoint, or promote") ErrVolumeBusy = errors.New("volume is held by another claim") - ErrReplicaPresent = errors.New("writable volume intent recorded; waiting for replica retirement before retry") - ErrVolumePendingCommit = errors.New("volume intent or replica retirement is not yet persisted; retry") + ErrReplicaPresent = errors.New("live writable attachment requires replica and recovery checkpoint retirement") + ErrReplicatingPool = errors.New("live writable volumes require a non-replicating pool") // Replaying a journal takes a writable mount, so readers stay out. ErrVolumeNeedsRecovery = errors.New("volume needs recovery by a writable claim") diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 7b1510b9..d51ff95f 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -3,6 +3,7 @@ package pool import ( "context" "encoding/json/v2" + "errors" "fmt" "os" "path/filepath" @@ -330,7 +331,9 @@ func (m *Manager) reclaimOrphanArchiveCks(ctx context.Context, claims map[string continue } if err := m.deleteArchiveCk(ctx, ckpt.ID); err != nil { - logger.Warnf(ctx, "reclaim %s: %v", ckpt.ID, err) + if !errors.Is(err, errCheckpointReferenced) { + logger.Warnf(ctx, "reclaim %s: %v", ckpt.ID, err) + } } else { logger.Infof(ctx, "reclaimed orphan archive ck %s", ckpt.ID) } diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index b8e04f23..77681465 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -243,32 +243,9 @@ func (m *Manager) clearReplicaIntent(ctx context.Context, sb *types.Sandbox, ck func (m *Manager) retireReplica(ctx context.Context, sb *types.Sandbox) { sb.Transition.Lock() defer sb.Transition.Unlock() - err := m.dropReplica(ctx, sb) - if err != nil && !benignSweepErr(err) { + if err := m.dropReplica(ctx, sb); err != nil && !benignSweepErr(err) { log.WithFunc("pool.retireReplica").Warnf(ctx, "stop replicating %s: %v", sb.ID, err) } - m.mu.Lock() - defer m.mu.Unlock() - if m.claimed[sb.ID] != sb { - return - } - for _, v := range sb.VolumesWant { - if !pendingVolumeWriter(sb, v) { - continue - } - if err != nil { - if m.volumeErrors[sb] == nil { - m.volumeErrors[sb] = map[string]string{} - } - message := volumeReplicaFailed - if errors.Is(err, ErrArchiveStoreDown) { - message = volumeReplicaUnavailable - } - m.volumeErrors[sb][v.Name] = message - } else if message := m.volumeErrors[sb][v.Name]; message == volumeReplicaFailed || message == volumeReplicaUnavailable { - delete(m.volumeErrors[sb], v.Name) - } - } } // dropReplica ends sb's replication; the caller holds sb.Transition. diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 0334276c..43a46a1e 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -14,10 +14,8 @@ import ( ) const ( - volumeChangeFailed = "volume change failed; retry the desired target" - volumeChangeInterrupted = "volume change interrupted; retry the desired target" - volumeReplicaFailed = "replica retirement failed; background retry pending" - volumeReplicaUnavailable = "replica registry unavailable; background retry pending" + volumeChangeFailed = "volume change failed; retry the desired target" + volumeChangeInterrupted = "volume change interrupted; retry the desired target" ) type volumeEdit func(current, shapeContext []types.Volume) ([]types.Volume, error) @@ -118,6 +116,9 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } + if err = m.checkLiveAdditions(sb, additions); err != nil { + return view, err + } sb.Touch() m.mu.Lock() uncertain := volumeDifferenceByName(volumeDifferenceByName(sb.VolumesWant, sb.Volumes), target) @@ -132,9 +133,6 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v if err = m.prepareVolumeTarget(ctx, sb, run, target, additions, resolved); err != nil { return view, err } - if err = m.checkVolumeApply(ctx, sb, run); err != nil { - return view, err - } m.mu.Lock() removals := volumeDifferenceByName(sb.Volumes, target) m.mu.Unlock() @@ -153,7 +151,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v m.mu.Unlock() for _, v := range pending { failedName = v.Name - if err = m.checkVolumeApply(ctx, sb, run); err != nil { + if err = m.checkVolumeApply(ctx, sb, run, v); err != nil { return view, err } repair, resolveErr := m.resolveCatalogVolume(ctx, v) @@ -307,6 +305,11 @@ func (m *Manager) admitVolumeTarget(ctx context.Context, sb *types.Sandbox, run if sb.OnExpire == types.ExpireArchive && len(target) > 0 { return ErrVolumeCapture } + if slices.ContainsFunc(additions, types.Volume.RW) { + if err := m.liveWriterError(sb); err != nil { + return err + } + } if err := m.reserveResolved(run.access, resolved, sb); err != nil { return err } @@ -314,28 +317,36 @@ func (m *Manager) admitVolumeTarget(ctx context.Context, sb *types.Sandbox, run return nil } -// checkVolumeApply keeps desired writers reserved while the scan retires a replica. -// The claim snapshot must acknowledge retirement before a device can be exposed. -func (m *Manager) checkVolumeApply(ctx context.Context, sb *types.Sandbox, run *volumeRun) error { +func (m *Manager) checkLiveAdditions(sb *types.Sandbox, additions []types.Volume) error { + m.mu.Lock() + defer m.mu.Unlock() + if slices.ContainsFunc(additions, types.Volume.RW) { + return m.liveWriterError(sb) + } + return nil +} + +func (m *Manager) checkVolumeApply(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume) error { m.mu.Lock() defer m.mu.Unlock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { return err } - if !slices.ContainsFunc(sb.VolumesWant, func(v types.Volume) bool { return pendingVolumeWriter(sb, v) }) { - return nil - } - if sb.HasReplica() { - return ErrReplicaPresent - } - if !m.store.synced() { - return ErrVolumePendingCommit + if v.RW() { + return m.liveWriterError(sb) } return nil } -func pendingVolumeWriter(sb *types.Sandbox, v types.Volume) bool { - return v.RW() && volumeIndex(sb.VolumesWant, v.Name) >= 0 && volumeIndex(sb.Volumes, v.Name) < 0 +// liveWriterError checks the policy and recoverable replicas of this claim; callers hold m.mu. +func (m *Manager) liveWriterError(sb *types.Sandbox) error { + if m.view.Load().poolReplicate[sb.PolicyKey()] > 0 { + return ErrReplicatingPool + } + if sb.HasReplica() || len(m.store.checkpointRefs(sb.ID)) > 0 { + return ErrReplicaPresent + } + return nil } func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume) error { @@ -431,16 +442,9 @@ func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { state = types.VolumePending } message := m.volumeErrors[sb][v.Name] - if pendingVolumeWriter(sb, v) { - switch { - case !m.store.synced(): - message = ErrVolumePendingCommit.Error() - case sb.HasReplica(): - if message != volumeReplicaFailed && message != volumeReplicaUnavailable { - message = ErrReplicaPresent.Error() - } - case message == volumeReplicaFailed || message == volumeReplicaUnavailable: - message = "" + if state == types.VolumePending && v.RW() { + if err := m.liveWriterError(sb); err != nil { + message = err.Error() } } if message == "" && state != types.VolumeAttached && !view.Converging { @@ -462,7 +466,7 @@ func (m *Manager) recordVolumeError(sb *types.Sandbox, run *volumeRun, name stri return } errs := m.volumeErrors[sb] - if name != "" && !errors.Is(err, ErrReplicaPresent) && !errors.Is(err, ErrVolumePendingCommit) { + if name != "" && !errors.Is(err, ErrReplicaPresent) && !errors.Is(err, ErrReplicatingPool) { message := volumeChangeFailed if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { message = volumeChangeInterrupted diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index c4d5dfae..3c747fca 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -1,7 +1,6 @@ package pool import ( - "context" "errors" "fmt" "slices" @@ -13,42 +12,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestLiveWriterConvergesThroughReplicaScans(t *testing.T) { - for _, pending := range []bool{false, true} { - t.Run(fmt.Sprintf("pending=%t", pending), func(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - if pending { - m.archives = &hookRegistry{Registry: c.reg, losePutReply: true} - if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { - t.Fatal(err) - } - m.archives = c.reg - } else { - replicateNow(t, m, sb) - } - writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - var view types.SandboxVolumesResponse - var err error - for cycle := range 5 { - view, err = m.SetVolumes(t.Context(), sb.ID, "", writer) - if err == nil { - if len(view.Volumes) != 1 || sb.HasReplica() { - t.Fatalf("writer exposed alongside replica: %+v, replica=%t", view, sb.HasReplica()) - } - } - t.Logf("cycle %d: wanted=%+v applied=%+v states=%+v replica=%t synced=%t error=%v", cycle, view.VolumesWant, view.Volumes, view.States, sb.HasReplica(), m.store.synced(), err) - m.replicas = map[string]replicaState{sb.ID: {tried: time.Now().Add(-2 * time.Duration(replicateEvery) * time.Second)}} - m.replicateOnce(t.Context()) - waitFor(t, func() bool { return !m.replicating.Load() }) - } - if err != nil { - t.Fatalf("writer never converged after five replica scans: %+v, %v", view, err) - } - }) - } -} - func TestReplicaRetirementKeepsTombstoneUntilClaimCommit(t *testing.T) { m, c := newReplicaVolumeManager(t) sb := mustClaim(t, m, testKey) @@ -96,15 +59,11 @@ func TestReplicaUnpublishedIntentSurvivesFailedClearAndRestart(t *testing.T) { t.Fatal("missing unpublished checkpoint") } m.archives = c.reg - writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { - t.Fatal(err) - } m.store.recommitting.Store(true) m.store.replaceFile = func([]byte) error { return errors.New("injected pending clear failure") } m.retireReplica(t.Context(), sb) persisted, err := newClaimStore(m.dataDir, false).load() - if err != nil || persisted[sb.ID].ReplicaPending != pending || len(persisted[sb.ID].VolumesWant) != 1 { + if err != nil || persisted[sb.ID].ReplicaPending != pending || len(persisted[sb.ID].VolumesWant) != 0 { t.Fatalf("failure did not retain old disk state: %+v, %v", persisted, err) } if !ckExists(t, m, pending) { @@ -116,11 +75,12 @@ func TestReplicaUnpublishedIntentSurvivesFailedClearAndRestart(t *testing.T) { if !ok { t.Fatal("restart mistook a failed pending clear for adoption and discarded the claim") } - if len(restored.VolumesWant) != 1 || len(restored.Volumes) != 0 || volumeHoldersOf(restarted, "data").writers != 1 { + if len(restored.VolumesWant) != 0 || len(restored.Volumes) != 0 { t.Fatalf("restart lost pending writer: %+v", restored) } - if view, err := restarted.SetVolumes(t.Context(), sb.ID, "", writer); err != nil || len(view.Volumes) != 1 { - t.Fatalf("retry after restart: %+v, %v", view, err) + restarted.retireReplica(t.Context(), restored) + if restored.HasReplica() { + t.Fatal("restart failed to retire unpublished replica") } } @@ -151,6 +111,11 @@ func TestReplicaPendingCheckpointCleanupWaitsForAcknowledgment(t *testing.T) { if err := m.store.commit(m.store.mark()); err != nil { t.Fatal(err) } + other := mustClaim(t, m, testKey) + m.mu.Lock() + other.LeaseSeconds++ + m.store.set(other) + m.mu.Unlock() m.retryArchiveDeletes(t.Context()) if ckExists(t, m, pending) || m.deleteMarked(pending) { t.Fatal("acknowledged cleanup leaked checkpoint or retry marker") @@ -216,201 +181,6 @@ func TestReplicaRecoveryDoesNotRepublishVolumeIntent(t *testing.T) { } } -func TestLiveWriterIntentSurvivesRestartBeforeRetirement(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - replicateNow(t, m, sb) - writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { - t.Fatal(err) - } - persisted, err := newClaimStore(m.dataDir, false).load() - if err != nil || !slices.Equal(persisted[sb.ID].VolumesWant, writer) || len(persisted[sb.ID].Volumes) != 0 { - t.Fatalf("durable intent: %+v, %v", persisted, err) - } - // Consume without acknowledging it locally, then restart before the next scan. - if _, err = c.reg.Consume(t.Context(), sb.ID, sb.ArchiveEpoch); err != nil { - t.Fatal(err) - } - restarted := restartReplicaVolumeManager(t, m, c) - restored, ok := restarted.byID(sb.ID) - if !ok { - t.Fatal("pending writer lost after restart") - } - view, err := restarted.GetVolumes(sb.ID, "") - if err != nil || len(view.VolumesWant) != 1 || len(view.Volumes) != 0 || view.Converging || view.States[0].Error != ErrReplicaPresent.Error() { - t.Fatalf("restart view: %+v, %v", view, err) - } - t.Logf("restart: %+v replica=%t synced=%t", view, restored.HasReplica(), restarted.store.synced()) - if _, err = restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: writer}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("second writer after restart: %v", err) - } - rec, _, _ := c.reg.Get(t.Context(), sb.ID) - restarted.reconcileOwnRow(t.Context(), rec, time.Now()) - if rec, _, err = c.reg.Get(t.Context(), sb.ID); err != nil || !rec.Consumed { - t.Fatalf("sweep rearmed replica: %+v, %v", rec, err) - } - restarted.replicateOnce(t.Context()) - waitFor(t, func() bool { return !restarted.replicating.Load() }) - view, err = restarted.GetVolumes(sb.ID, "") - if err != nil || restored.HasReplica() || len(view.Volumes) != 0 || view.Converging { - t.Fatalf("retirement unexpectedly attached: %+v, %v", view, err) - } - t.Logf("retired: %+v replica=%t synced=%t", view, restored.HasReplica(), restarted.store.synced()) - if view, err = restarted.SetVolumes(t.Context(), sb.ID, "", writer); err != nil || len(view.Volumes) != 1 { - t.Fatalf("explicit retry: %+v, %v", view, err) - } - t.Logf("attached on retry: %+v replica=%t", view, restored.HasReplica()) -} - -func TestLiveWriterRetirementDiagnosticsSurviveRetry(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - replicateNow(t, m, sb) - writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { - t.Fatal(err) - } - m.archives = &hookRegistry{Registry: c.reg, loseConsumeReply: true} - m.retireReplica(t.Context(), sb) - for range 2 { - view, err := m.SetVolumes(t.Context(), sb.ID, "", writer) - if !errors.Is(err, ErrReplicaPresent) || len(view.Volumes) != 0 || view.States[0].Error != volumeReplicaUnavailable { - t.Fatalf("retirement failure diagnostic: %+v, %v", view, err) - } - t.Logf("retire failure and retry: %+v replica=%t synced=%t", view, sb.HasReplica(), m.store.synced()) - } - m.archives = c.reg - m.retireReplica(t.Context(), sb) - if view, err := m.SetVolumes(t.Context(), sb.ID, "", writer); err != nil || len(view.Volumes) != 1 || view.States[0].Error != "" { - t.Fatalf("retry after registry recovery: %+v, %v", view, err) - } -} - -func TestLiveWriterFailedIntentDoesNotApply(t *testing.T) { - for _, cancelAfterWrite := range []bool{false, true} { - t.Run(fmt.Sprintf("cancel=%t", cancelAfterWrite), func(t *testing.T) { - m, _ := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - replicateNow(t, m, sb) - eng := m.eng.(*fakeEngine) - before := eng.volumeOpsLog() - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - m.store.recommitting.Store(true) - m.store.replaceFile = func(raw []byte) error { - if cancelAfterWrite { - if err := m.store.replace(raw); err != nil { - return err - } - cancel() - return nil - } - return errors.New("injected intent write failure") - } - writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - view, err := m.SetVolumes(ctx, sb.ID, "", writer) - if err == nil || len(view.VolumesWant) != 1 || len(view.Volumes) != 0 || !slices.Equal(before, eng.volumeOpsLog()) { - t.Fatalf("failed intent applied: %+v, %v", view, err) - } - t.Logf("failed/canceled intent: %+v replica=%t synced=%t error=%v", view, sb.HasReplica(), m.store.synced(), err) - if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: writer}); !errors.Is(err, ErrVolumeBusy) { - t.Fatalf("failure lost reservation: %v", err) - } - m.store.replaceFile = nil - if _, err = m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { - t.Fatalf("remove canceled/failed target: %v", err) - } - if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { - t.Fatalf("remove retained intent: %+v", got) - } - }) - } -} - -func TestLiveWriterWaitsForRetirementCommitAcknowledgment(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - replicateNow(t, m, sb) - writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { - t.Fatal(err) - } - before := m.eng.(*fakeEngine).volumeOpsLog() - m.store.recommitting.Store(true) - m.store.replaceFile = func([]byte) error { return errors.New("injected retirement persistence failure") } - m.retireReplica(t.Context(), sb) - view, err := m.GetVolumes(sb.ID, "") - if err != nil || sb.HasReplica() || m.store.synced() || len(view.States) != 1 || view.States[0].Error != ErrVolumePendingCommit.Error() { - t.Fatalf("unacknowledged clear: %+v, %v", view, err) - } - t.Logf("unacknowledged retirement: %+v replica=%t synced=%t", view, sb.HasReplica(), m.store.synced()) - if _, err = m.SetVolumes(t.Context(), sb.ID, "", writer); err == nil { - t.Fatal("failed persistence acknowledged attachment") - } - if !slices.Equal(before, m.eng.(*fakeEngine).volumeOpsLog()) { - t.Fatal("writer exposed before retirement persisted") - } - rec, _, err := c.reg.Get(t.Context(), sb.ID) - if err != nil || !rec.Consumed { - t.Fatalf("retirement tombstone: %+v, %v", rec, err) - } - m.reconcileOwnRow(t.Context(), rec, time.Now()) - if _, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || !ok { - t.Fatalf("sweep erased pending retirement: %t, %v", ok, getErr) - } - // Any newer whole-claim snapshot can acknowledge the clear; no special retry ticket is needed. - m.store.replaceFile = nil - if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { - t.Fatal(err) - } - m.reconcileOwnRow(t.Context(), rec, time.Now()) - if view, err = m.SetVolumes(t.Context(), sb.ID, "", writer); err != nil || len(view.Volumes) != 1 { - t.Fatalf("retry after newer acknowledgment: %+v, %v", view, err) - } - t.Logf("newer snapshot acknowledged retirement: %+v replica=%t synced=%t", view, sb.HasReplica(), m.store.synced()) -} - -func TestLiveWriterApplyGuardRequiresAcknowledgedRetirement(t *testing.T) { - m, _ := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - replicateNow(t, m, sb) - writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { - t.Fatal(err) - } - // Pause a request after its target commit, before its physical-apply check. - ctx, run, err := m.beginVolumeRun(t.Context(), sb, "") - if err != nil { - t.Fatal(err) - } - defer m.finishVolumeRun(sb, run) - if err = m.prepareVolumeTarget(ctx, sb, run, writer, nil, nil); err != nil { - t.Fatal(err) - } - before := m.eng.(*fakeEngine).volumeOpsLog() - m.store.recommitting.Store(true) - m.store.replaceFile = func([]byte) error { return errors.New("injected retirement clear failure") } - m.retireReplica(t.Context(), sb) - if sb.HasReplica() || m.store.synced() { - t.Fatal("expected memory clear with unacknowledged snapshot") - } - if err = m.checkVolumeApply(ctx, sb, run); !errors.Is(err, ErrVolumePendingCommit) { - t.Fatalf("physical apply guard passed unacknowledged clear: %v", err) - } - if !slices.Equal(before, m.eng.(*fakeEngine).volumeOpsLog()) { - t.Fatal("device call before acknowledgment") - } - t.Logf("apply boundary: want=%+v applied=%+v replica=%t synced=%t error=%v", sb.VolumesWant, sb.Volumes, sb.HasReplica(), m.store.synced(), err) - m.store.replaceFile = nil - if err = m.store.commit(m.store.mark()); err != nil { - t.Fatal(err) - } - if err = m.checkVolumeApply(ctx, sb, run); err != nil { - t.Fatalf("acknowledged clear still blocked: %v", err) - } -} - func TestReplicaConsumedRowPurgeWaitsForSnapshot(t *testing.T) { for _, claimed := range []bool{false, true} { t.Run(fmt.Sprintf("claimed=%t", claimed), func(t *testing.T) { @@ -446,57 +216,13 @@ func TestReplicaConsumedRowPurgeWaitsForSnapshot(t *testing.T) { } } -func TestLiveWriterWaitingIntentCanRelease(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - replicateNow(t, m, sb) - writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { - t.Fatal(err) - } - if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { - t.Fatal(err) - } - waitFor(t, func() bool { return volumeHoldersOf(m, "data") == (volumeHolders{}) }) - if _, ok := m.byID(sb.ID); ok { - t.Fatal("released pending claim remains") - } - if rec, ok, err := c.reg.Get(t.Context(), sb.ID); err != nil || ok && !rec.Consumed { - t.Fatalf("released replica still adoptable: %+v, %t, %v", rec, ok, err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: writer}); err != nil { - t.Fatalf("released intent retained writer admission: %v", err) - } -} - -func TestLiveWriterIntentIsDroppedWhenReplicaMoves(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - replicateNow(t, m, sb) - if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}}); !errors.Is(err, ErrReplicaPresent) { - t.Fatal(err) - } - c.expire("a") - other := c.node(t, "b", t.TempDir(), keepAnHour) - if adopted, err := other.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { - t.Fatalf("adopt: %t, %v", adopted, err) - } - m.retireReplica(t.Context(), sb) - if _, ok := m.byID(sb.ID); ok { - t.Fatal("old owner kept a pending writer after replica moved") - } - moved, ok := other.byID(sb.ID) - if !ok || len(moved.VolumesWant) != 0 || len(moved.Volumes) != 0 { - t.Fatalf("old replica carried new volume intent: %+v", moved) - } -} - -func TestLiveWriterRecordsIntentBeforeRetiringReplica(t *testing.T) { +func TestLiveWriterRefusesExistingReplica(t *testing.T) { for _, pending := range []bool{false, true} { for _, attachOnly := range []bool{false, true} { for _, patch := range []bool{false, true} { t.Run(fmt.Sprintf("pending=%t/attach_only=%t/patch=%t", pending, attachOnly, patch), func(t *testing.T) { m, c := newReplicaVolumeManager(t) + disableReplication(m) sb := mustClaim(t, m, testKey) if pending { m.archives = &hookRegistry{Registry: c.reg, losePutReply: true} @@ -522,7 +248,7 @@ func TestLiveWriterRecordsIntentBeforeRetiringReplica(t *testing.T) { if !errors.Is(err, ErrReplicaPresent) { t.Fatalf("writer admitted while replica is recoverable: %v", err) } - if len(view.Volumes) != 0 || len(view.VolumesWant) != 1 || view.Converging || !slices.Equal(before, eng.volumeOpsLog()) || volumeHoldersOf(m, "data").writers != 1 { + if len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || view.Converging || !slices.Equal(before, eng.volumeOpsLog()) || volumeHoldersOf(m, "data").writers != 0 { t.Fatalf("refused writer changed devices or holds: %+v", view) } if next, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || !ok || next.Checkpoint != row.Checkpoint || next.Consumed { @@ -544,6 +270,7 @@ func TestLiveWriterRecordsIntentBeforeRetiringReplica(t *testing.T) { func TestReplicaPublicationRefusesVolumeAdmittedDuringExport(t *testing.T) { m, c := newReplicaVolumeManager(t) + disableReplication(m) sb := mustClaim(t, m, testKey) stalled := &stallingStore{Store: m.ckpts, published: make(chan string), release: make(chan struct{})} m.ckpts = stalled @@ -574,6 +301,7 @@ func TestReplicaPublicationRefusesVolumeAdmittedDuringExport(t *testing.T) { func TestReplicaCaptureRefusesPendingLiveWriter(t *testing.T) { m, c := newReplicaVolumeManager(t) + disableReplication(m) sb := mustClaim(t, m, testKey) eng := m.eng.(*fakeEngine) entered, resume := make(chan struct{}), make(chan struct{}) @@ -648,7 +376,6 @@ func newReplicaVolumeManager(t *testing.T) (*Manager, *failoverCluster) { func restartReplicaVolumeManager(t *testing.T, old *Manager, c *failoverCluster) *Manager { t.Helper() spec := archivePool(keepAnHour) - spec.ReplicateSeconds = c.replicate cfg := &config.Config{APIToken: "root", DataDir: old.dataDir, CheckpointDir: c.ckDir, Pools: []config.PoolSpec{spec}, Mesh: &config.MeshConfig{NodeID: old.nodeID}, Volumes: []config.VolumeSpec{{Name: "data", Path: old.catalog.entries["data"].disk.Path, Writable: true}}} backends := testBackends(t, cfg) backends.Archives = c.reg @@ -662,3 +389,9 @@ func restartReplicaVolumeManager(t *testing.T, old *Manager, c *failoverCluster) m.liftFence() return m } + +func disableReplication(m *Manager) { + view := *m.view.Load() + view.poolReplicate = nil + m.view.Store(&view) +} diff --git a/sandboxd/pool/volume_replication_policy_test.go b/sandboxd/pool/volume_replication_policy_test.go index f6585295..e89873ae 100644 --- a/sandboxd/pool/volume_replication_policy_test.go +++ b/sandboxd/pool/volume_replication_policy_test.go @@ -2,11 +2,14 @@ package pool import ( "errors" + "fmt" + "slices" "sync" "sync/atomic" "testing" "time" + "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -105,3 +108,193 @@ func TestArchiveCheckpointCleanupIgnoresUnrelatedClaimWrites(t *testing.T) { t.Fatal("unreferenced checkpoint or delete marker remained") } } + +func TestLiveWriterRequiresNonReplicatingPool(t *testing.T) { + for _, patch := range []bool{false, true} { + for _, attachOnly := range []bool{false, true} { + t.Run(fmt.Sprintf("patch=%t/attach_only=%t", patch, attachOnly), func(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + before := m.store.mark() + beforeOps := m.eng.(*fakeEngine).volumeOpsLog() + t.Logf("before live mutation: snapshot=%+v device_ops=%v", before, beforeOps) + want := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: attachOnly}} + var view types.SandboxVolumesResponse + var err error + if patch { + view, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: want}) + } else { + view, err = m.SetVolumes(t.Context(), sb.ID, "", want) + } + if !errors.Is(err, ErrReplicatingPool) { + t.Fatalf("replicating pool accepted live writer: %+v, %v", view, err) + } + if sb.HasReplica() || len(view.VolumesWant) != 0 || len(view.Volumes) != 0 || view.Converging || m.store.mark() != before || volumeHoldersOf(m, "data") != (volumeHolders{}) || !slices.Equal(beforeOps, m.eng.(*fakeEngine).volumeOpsLog()) { + t.Fatalf("policy refusal changed claim, device, or reservation: %+v snapshot=%+v->%+v ops=%v->%v", view, before, m.store.mark(), beforeOps, m.eng.(*fakeEngine).volumeOpsLog()) + } + }) + } + } +} + +func TestInitialWriterRemainsAvailableInReplicatingPool(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}) + if err != nil || len(sb.Volumes) != 1 { + t.Fatalf("initial writer: %+v, %v", sb, err) + } + m.replicateOnce(t.Context()) + waitFor(t, func() bool { return !m.replicating.Load() }) + if sb.HasReplica() || len(m.eng.(*fakeEngine).snapSaves) != 0 { + t.Fatal("replicated an initial writer") + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", sb.Volumes); err != nil { + t.Fatalf("held writer retry refused: %v", err) + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("writer cleanup refused: %v", err) + } +} + +func TestLiveWriterUsesItsPolicySource(t *testing.T) { + for _, sourceReplicates := range []bool{false, true} { + t.Run(fmt.Sprint(sourceReplicates), func(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + other := testKey + other.Template = "other:template" + m.mu.Lock() + if sourceReplicates { + sb.Key, sb.PolicySource = other, testKey + } else { + sb.PolicySource = other + } + m.mu.Unlock() + _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}) + if sourceReplicates && !errors.Is(err, ErrReplicatingPool) || !sourceReplicates && err != nil { + t.Fatalf("policy source replication=%t: %v", sourceReplicates, err) + } + }) + } +} + +func TestLiveWriterWaitsOnlyForItsOwnCheckpointReferences(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + other := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + replicateNow(t, m, other) + m = restartReplicaVolumeManager(t, m, c) + sb, _ = m.byID(sb.ID) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { + t.Fatalf("restart ignored actual replica: %v", err) + } + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected clear failure") } + m.retireReplica(t.Context(), sb) + before := m.store.mark() + if sb.HasReplica() || len(m.store.checkpointRefs(sb.ID)) == 0 { + t.Fatal("expected an acknowledged replica reference after memory clear") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) || m.store.mark() != before { + t.Fatalf("unacknowledged replica clear allowed writer or staged intent: %v", err) + } + m.store.replaceFile = nil + if err := m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + if len(m.store.checkpointRefs(other.ID)) == 0 { + t.Fatal("unrelated replica did not survive the restart") + } + if view, err := m.SetVolumes(t.Context(), sb.ID, "", writer); err != nil || len(view.Volumes) != 1 { + t.Fatalf("acknowledged retirement blocked writer: %+v, %v", view, err) + } +} + +func TestLegacyWriterIntentCanRemoveReadOnlyVolumes(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.mu.Lock() + sb.VolumesWant = append(slices.Clone(sb.Volumes), types.Volume{Name: "other", Mode: types.VolumeModeRW, AttachOnly: true}) + snap := m.store.set(sb) + m.mu.Unlock() + if err := m.store.commit(snap); err != nil { + t.Fatal(err) + } + c := newReplicaCluster(t) + spec := archivePool(keepAnHour) + spec.ReplicateSeconds = c.replicate + cfg := &config.Config{APIToken: "root", DataDir: m.dataDir, CheckpointDir: c.ckDir, Volumes: catalog, Pools: []config.PoolSpec{spec}, Mesh: &config.MeshConfig{NodeID: "a"}} + backends := testBackends(t, cfg) + backends.Archives = c.reg + var err error + m, err = NewManager(t.Context(), cfg, eng, testSecrets(t), backends) + if err != nil { + t.Fatal(err) + } + c.renew(t, "a") + if err = m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + m.liftFence() + got, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Remove: []string{"data"}}) + if !errors.Is(err, ErrReplicatingPool) || len(got.Volumes) != 0 || len(got.VolumesWant) != 1 || got.VolumesWant[0].Name != "other" { + t.Fatalf("legacy writer blocked reader cleanup: %+v, %v", got, err) + } + if got, err = m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil || len(got.VolumesWant) != 0 || len(got.Volumes) != 0 { + t.Fatalf("legacy writer cannot be canceled: %+v, %v", got, err) + } + if volumeHoldersOf(m, "other") != (volumeHolders{}) { + t.Fatal("canceled legacy writer retained admission") + } +} + +func TestLegacyWriterApplyWaitsForItsRetirementReference(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + m.mu.Lock() + sb.VolumesWant = writer + snap := m.store.set(sb) + m.mu.Unlock() + if err := m.store.commit(snap); err != nil { + t.Fatal(err) + } + m = restartReplicaVolumeManager(t, m, c) + sb, _ = m.byID(sb.ID) + ctx, run, err := m.beginVolumeRun(t.Context(), sb, "") + if err != nil { + t.Fatal(err) + } + defer m.finishVolumeRun(sb, run) + if err = m.prepareVolumeTarget(ctx, sb, run, writer, nil, nil); err != nil { + t.Fatal(err) + } + before := m.eng.(*fakeEngine).volumeOpsLog() + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected legacy retirement failure") } + m.retireReplica(t.Context(), sb) + if sb.HasReplica() || len(m.store.checkpointRefs(sb.ID)) == 0 { + t.Fatal("expected only an unacknowledged retirement reference") + } + if err = m.checkVolumeApply(ctx, sb, run, writer[0]); !errors.Is(err, ErrReplicaPresent) { + t.Fatalf("legacy apply ignored its retirement reference: %v", err) + } + if !slices.Equal(before, m.eng.(*fakeEngine).volumeOpsLog()) { + t.Fatal("legacy writer called the device before retirement acknowledgement") + } + m.store.replaceFile = nil + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + if err = m.checkVolumeApply(ctx, sb, run, writer[0]); err != nil { + t.Fatalf("acknowledged legacy retirement blocked apply: %v", err) + } +} diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go index 64f66e57..7dd90ce3 100644 --- a/sandboxd/pool/volume_retirement_test.go +++ b/sandboxd/pool/volume_retirement_test.go @@ -219,7 +219,7 @@ func TestClaimRecoveryAttemptsRetainOnlyDistinctReferences(t *testing.T) { t.Fatal("expected failed snapshot") } } - if len(s.attemptRefs) != 1 || len(s.attemptRefs[sb.ID]) != 2 { + if len(s.attemptRefs) != 1 || len(s.attemptRefs[sb.ID].volumes) != 2 { t.Fatalf("renewals retained duplicate history: %+v", s.attemptRefs) } refs := s.recoveryVolumes(sb.ID)[sb.ID] diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index ebdd5a81..57201f86 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -68,7 +68,7 @@ var poolErrHTTP = []struct { {pool.ErrVolumeCapture, http.StatusConflict, ""}, {pool.ErrVolumeBusy, http.StatusConflict, ""}, {pool.ErrReplicaPresent, http.StatusConflict, ""}, - {pool.ErrVolumePendingCommit, http.StatusServiceUnavailable, ""}, + {pool.ErrReplicatingPool, http.StatusConflict, ""}, {pool.ErrVolumeNeedsRecovery, http.StatusConflict, ""}, {pool.ErrArchived, http.StatusConflict, ""}, {pool.ErrPaused, http.StatusConflict, ""}, diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 45a3dee1..0c3cd4be 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -153,7 +153,7 @@ func TestLiveVolumeRoutesHideOtherTenants(t *testing.T) { func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { const cause = "cannot unmount /data; close guest references and retry" - for _, refusal := range []error{pool.ErrVolumeBusy, pool.ErrVolumeCapture, pool.ErrReplicaPresent} { + for _, refusal := range []error{pool.ErrVolumeBusy, pool.ErrVolumeCapture, pool.ErrReplicaPresent, pool.ErrReplicatingPool} { t.Run(refusal.Error(), func(t *testing.T) { mgr := &fakeManager{setVolumes: func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) { return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "old"}}, States: []types.VolumeState{{Name: "old", State: "pending", Error: cause}}, Error: "volume change failed; retry the desired target"}, fmt.Errorf("%w: volume %q", refusal, "data") From 6ba4d49ddff8e9fc987f1792c805da43785da066 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 07:46:44 +0000 Subject: [PATCH 57/72] pool: reproduce replica cleanup gaps after failover disablement --- sandboxd/pool/replica_cleanup_test.go | 93 +++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 sandboxd/pool/replica_cleanup_test.go diff --git a/sandboxd/pool/replica_cleanup_test.go b/sandboxd/pool/replica_cleanup_test.go new file mode 100644 index 00000000..b02be705 --- /dev/null +++ b/sandboxd/pool/replica_cleanup_test.go @@ -0,0 +1,93 @@ +package pool + +import ( + "errors" + "fmt" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestReconcileRetiresReplicasWhenFailoverDisabled(t *testing.T) { + for _, pending := range []bool{false, true} { + t.Run(fmt.Sprintf("pending=%t", pending), func(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + if pending { + m.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + } else { + replicateNow(t, m, sb) + } + ck := sb.ReplicaCk + if pending { + ck = sb.ReplicaPending + } + cfg := &config.Config{ + APIToken: "root", DataDir: m.dataDir, CheckpointDir: c.ckDir, + Pools: []config.PoolSpec{{PoolKey: testKey}}, + Volumes: []config.VolumeSpec{{Name: "data", Path: m.catalog.entries["data"].disk.Path, Writable: true}}, + } + restarted, err := NewManager(t.Context(), cfg, m.eng, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + if restarted.archives != nil { + t.Fatal("failover-disabled fixture unexpectedly has a registry") + } + if err = restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + restored, ok := restarted.byID(sb.ID) + if !ok { + t.Fatal("disabling failover lost the running claim") + } + if restored.HasReplica() || restored.ArchiveEpoch != 0 { + t.Errorf("disabled failover retained replica fields: replica=%q pending=%q epoch=%d", restored.ReplicaCk, restored.ReplicaPending, restored.ArchiveEpoch) + } + want := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + for range 3 { + restarted.replicateOnce(t.Context()) + } + if view, applyErr := restarted.SetVolumes(t.Context(), sb.ID, "", want); applyErr != nil || len(view.Volumes) != 1 { + t.Errorf("disabled failover left live writer permanently blocked: %+v, %v", view, applyErr) + } + if ckExists(t, restarted, ck) && !restarted.deleteMarked(ck) { + t.Error("disabled failover left the retired checkpoint without a cleanup marker") + } + }) + } +} + +func TestReplicaFailedClearQueuesCheckpointCleanup(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + ck := sb.ReplicaCk + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected replica clear failure") } + sb.Transition.Lock() + err := m.dropReplica(t.Context(), sb) + sb.Transition.Unlock() + if err == nil || sb.HasReplica() { + t.Fatalf("expected an unacknowledged memory clear: replica=%t, %v", sb.HasReplica(), err) + } + if !m.deleteMarked(ck) { + t.Error("failed replica clear did not queue its checkpoint for cleanup") + } + m.retryArchiveDeletes(t.Context()) + if !ckExists(t, m, ck) { + t.Fatal("queued cleanup deleted a checkpoint still referenced by claims.json") + } + m.store.replaceFile = nil + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + m.retryArchiveDeletes(t.Context()) + if ckExists(t, m, ck) || m.deleteMarked(ck) { + t.Fatal("checkpoint cleanup did not finish after its reference retired") + } +} From b17f01d703de0ddc1a31d8072f2d702dda72b08f Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 07:58:53 +0000 Subject: [PATCH 58/72] pool: queue retired replicas across persistence failures --- docs/deploy.md | 16 +++ sandboxd/pool/archive.go | 17 ++- sandboxd/pool/claim.go | 3 +- sandboxd/pool/failover.go | 10 +- sandboxd/pool/reconcile.go | 35 ++++- sandboxd/pool/replica.go | 2 +- sandboxd/pool/replica_cleanup_test.go | 130 ++++++++++++++++-- sandboxd/pool/replica_test.go | 9 +- sandboxd/pool/volume_replica_test.go | 4 +- .../pool/volume_replication_policy_test.go | 4 +- 10 files changed, 188 insertions(+), 42 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index cbe90b6d..566e1bc7 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -167,6 +167,22 @@ fragment the warm pools): | `xlarge` | 4 | 8G | | `2xlarge` | 8 | 16G | +### Disabling archive failover + +Set `replicate_seconds: 0` while `archive_failover` remains enabled, restart, +and let the existing replicas and their recovery references retire before +disabling failover. Keep the old registry reachable during that transition. +Before removing it, drain its remaining claim rows. If adopters must be stopped +first, keep every adopter shut down or disabled until those rows are drained or +the adoption domain is permanently retired. Disabling the local registry does +not consume remote rows or fence another node. + +On startup without a registry, sandboxd clears replica fields from local running +or sleeping claims and queues their checkpoints for deletion before rewriting +the claims file. A failed rewrite keeps the affected checkpoints protected until +their references retire. Archived claims and unfinished archive transitions +retain their archive fields and checkpoints. + ### Reading sleeping claims without waking them A client that only lists or reads files (a UI showing an idle agent's diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 3a85a3d0..3bc7f2e4 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -18,8 +18,6 @@ import ( const archiveDeleteDir = "archive-deletes" -var errCheckpointReferenced = errors.New("checkpoint is referenced by a claim") - // archive*For resolve a key's thresholds (pool's, else node default); callers hold m.mu. func (m *Manager) archiveAfterFor(key types.PoolKey) time.Duration { if p, ok := m.activePool(key); ok { @@ -282,7 +280,7 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc } func (m *Manager) deleteOrphanArchiveCk(ctx context.Context, ckID string) { - if err := m.deleteArchiveCk(ctx, ckID); err != nil && !errors.Is(err, errCheckpointReferenced) { + if _, err := m.deleteArchiveCk(ctx, ckID); err != nil { log.WithFunc("pool.deleteOrphanArchiveCk").Warnf(ctx, "delete orphaned archive ck %s: %v", ckID, err) } } @@ -353,24 +351,25 @@ func (m *Manager) clearArchiveCk(ckID string) error { return err } -func (m *Manager) deleteArchiveCk(ctx context.Context, ckID string) error { +// deleteArchiveCk queues deletion and reports whether references allowed it to finish. +func (m *Manager) deleteArchiveCk(ctx context.Context, ckID string) (bool, error) { if err := m.markArchiveCk(ckID); err != nil { - return fmt.Errorf("track: %w", err) + return false, fmt.Errorf("track: %w", err) } unlock := m.lockRec(ckID) if _, referenced := m.store.checkpointRefs()[ckID]; referenced { unlock() - return errCheckpointReferenced + return false, nil } err := m.ckpts.Delete(ctx, ckID) unlock() if err != nil { - return err + return false, err } if err := m.clearArchiveCk(ckID); err != nil { - return fmt.Errorf("clear: %w", err) + return false, fmt.Errorf("clear: %w", err) } - return nil + return true, nil } func (m *Manager) retryArchiveDeletes(ctx context.Context) { diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index eb3f693e..03c2369c 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -4,7 +4,6 @@ import ( "cmp" "context" "crypto/subtle" - "errors" "fmt" "net" "slices" @@ -697,7 +696,7 @@ func (m *Manager) pauseExpired(ctx context.Context, sb *types.Sandbox) error { func (m *Manager) purgeArchiveCk(ctx context.Context, id, ck, tenant string) { logger := log.WithFunc("pool.purgeArchiveCk") - if err := m.deleteArchiveCk(ctx, ck); err != nil && !errors.Is(err, errCheckpointReferenced) { + if _, err := m.deleteArchiveCk(ctx, ck); err != nil { logger.Warnf(ctx, "delete archive ck %s: %v", ck, err) } m.counters.archiveDeletes.Add(1) diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index f0137626..5c9bb915 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -483,10 +483,12 @@ func (m *Manager) dropStaleRow(ctx context.Context, sb *types.Sandbox, rec archi func (m *Manager) purgeArchiveRow(ctx context.Context, rec archives.Record) { logger := log.WithFunc("pool.purgeArchiveRow") - if err := m.deleteArchiveCk(ctx, rec.Checkpoint); err != nil { - if !errors.Is(err, errCheckpointReferenced) { - logger.Warnf(ctx, "purge archive %s: delete ck %s: %v", rec.ID, rec.Checkpoint, err) - } + deleted, err := m.deleteArchiveCk(ctx, rec.Checkpoint) + if err != nil { + logger.Warnf(ctx, "purge archive %s: delete ck %s: %v", rec.ID, rec.Checkpoint, err) + return + } + if !deleted { return } switch err := m.archives.Delete(ctx, rec.ID, rec.Epoch); { diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index d51ff95f..d4072fac 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -3,7 +3,6 @@ package pool import ( "context" "encoding/json/v2" - "errors" "fmt" "os" "path/filepath" @@ -25,6 +24,9 @@ func (m *Manager) Reconcile(ctx context.Context) error { if err != nil { return err } + if err = m.retireReplicasWithoutRegistry(claims); err != nil { + return err + } fenced := m.fenceArchived(ctx, claims) vms, err := m.eng.List(ctx) if err != nil { @@ -131,6 +133,29 @@ func (m *Manager) Reconcile(ctx context.Context) error { return saveErr } +func (m *Manager) retireReplicasWithoutRegistry(claims map[string]*types.Sandbox) error { + if m.archives != nil { + return nil + } + var checkpoints []string + for _, sb := range claims { + if sb.ArchiveCk == "" && sb.ArchivePending == "" { + checkpoints = append(checkpoints, checkpointIDs(sb)...) + } + } + // Without a registry this manager has no node identity for later orphan scans. + // Queue cleanup before the last persisted replica reference can disappear. + for id, err := range m.markArchiveCks(checkpoints) { + return fmt.Errorf("queue replica without registry %s: %w", id, err) + } + for _, sb := range claims { + if sb.ArchiveCk == "" && sb.ArchivePending == "" { + sb.ReplicaCk, sb.ReplicaPending, sb.ArchiveEpoch = "", "", 0 + } + } + return nil +} + // sweepStaleVMs removes sbx-prefixed VMs no claim owns and returns the ones confirmed gone. func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRecord, owned map[string]bool) map[string]bool { var stale []string @@ -330,11 +355,9 @@ func (m *Manager) reclaimOrphanArchiveCks(ctx context.Context, claims map[string if row, ok := rows[ckpt.SandboxID]; ok && (row.Owner != m.nodeID || row.Checkpoint == ckpt.ID) { continue } - if err := m.deleteArchiveCk(ctx, ckpt.ID); err != nil { - if !errors.Is(err, errCheckpointReferenced) { - logger.Warnf(ctx, "reclaim %s: %v", ckpt.ID, err) - } - } else { + if deleted, err := m.deleteArchiveCk(ctx, ckpt.ID); err != nil { + logger.Warnf(ctx, "reclaim %s: %v", ckpt.ID, err) + } else if deleted { logger.Infof(ctx, "reclaimed orphan archive ck %s", ckpt.ID) } } diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 77681465..f6672e96 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -267,11 +267,11 @@ func (m *Manager) dropReplica(ctx context.Context, sb *types.Sandbox) error { if !live { return ErrUnknownSandbox } + m.deleteOrphanArchiveCk(ctx, ck) if err != nil { return fmt.Errorf("persist replica retirement: %w", err) } m.dropTombstone(ctx, sb.ID, consumed) - m.deleteOrphanArchiveCk(ctx, ck) return nil } diff --git a/sandboxd/pool/replica_cleanup_test.go b/sandboxd/pool/replica_cleanup_test.go index b02be705..753db22e 100644 --- a/sandboxd/pool/replica_cleanup_test.go +++ b/sandboxd/pool/replica_cleanup_test.go @@ -3,6 +3,8 @@ package pool import ( "errors" "fmt" + "os" + "path/filepath" "testing" "github.com/cocoonstack/sandbox/sandboxd/config" @@ -26,19 +28,9 @@ func TestReconcileRetiresReplicasWhenFailoverDisabled(t *testing.T) { if pending { ck = sb.ReplicaPending } - cfg := &config.Config{ - APIToken: "root", DataDir: m.dataDir, CheckpointDir: c.ckDir, - Pools: []config.PoolSpec{{PoolKey: testKey}}, - Volumes: []config.VolumeSpec{{Name: "data", Path: m.catalog.entries["data"].disk.Path, Writable: true}}, - } - restarted, err := NewManager(t.Context(), cfg, m.eng, testSecrets(t), testBackends(t, cfg)) - if err != nil { - t.Fatal(err) - } - if restarted.archives != nil { - t.Fatal("failover-disabled fixture unexpectedly has a registry") - } - if err = restarted.Reconcile(t.Context()); err != nil { + drainClaimRow(t, c, sb.ID) + restarted := newManagerWithoutRegistry(t, m, c) + if err := restarted.Reconcile(t.Context()); err != nil { t.Fatal(err) } restored, ok := restarted.byID(sb.ID) @@ -62,6 +54,89 @@ func TestReconcileRetiresReplicasWhenFailoverDisabled(t *testing.T) { } } +func TestDisabledReplicaCleanupSurvivesFailedStartupRewrite(t *testing.T) { + for _, renamed := range []bool{false, true} { + t.Run(fmt.Sprintf("renamed=%t", renamed), func(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + ck := sb.ReplicaCk + drainClaimRow(t, c, sb.ID) + restarted := newManagerWithoutRegistry(t, m, c) + restarted.store.replaceFile = func(raw []byte) error { + if renamed { + if err := restarted.store.replace(raw); err != nil { + return err + } + } + return errors.New("injected startup replacement failure") + } + if err := restarted.Reconcile(t.Context()); err == nil { + t.Fatal("startup replacement unexpectedly succeeded") + } + if !restarted.deleteMarked(ck) || !ckExists(t, restarted, ck) { + t.Fatal("failed startup lost a checkpoint or its cleanup intent") + } + persisted, err := newClaimStore(m.dataDir, false).load() + if err != nil || persisted[sb.ID] == nil || persisted[sb.ID].HasReplica() == renamed { + t.Fatalf("startup failure did not reach the expected persisted state: %+v, %v", persisted, err) + } + recovered := newManagerWithoutRegistry(t, restarted, c) + if err = recovered.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if restored, ok := recovered.byID(sb.ID); !ok || restored.HasReplica() || ckExists(t, recovered, ck) || recovered.deleteMarked(ck) { + t.Fatal("next startup did not retain the claim and finish queued cleanup") + } + }) + } +} + +func TestDisabledReplicaCleanupRequiresDurableMarker(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + ck := sb.ReplicaCk + drainClaimRow(t, c, sb.ID) + markerDir := filepath.Join(m.dataDir, archiveDeleteDir) + if err := os.WriteFile(markerDir, []byte("blocked"), 0o600); err != nil { + t.Fatal(err) + } + restarted := newManagerWithoutRegistry(t, m, c) + if err := restarted.Reconcile(t.Context()); err == nil { + t.Fatal("startup retired a replica without a durable cleanup marker") + } + persisted, err := newClaimStore(m.dataDir, false).load() + if err != nil || persisted[sb.ID] == nil || persisted[sb.ID].ReplicaCk != ck || !ckExists(t, restarted, ck) { + t.Fatalf("failed marker write changed the recoverable replica: %+v, %v", persisted, err) + } + if err = os.Remove(markerDir); err != nil { + t.Fatal(err) + } + if err = restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if ckExists(t, restarted, ck) { + t.Fatal("marker repair did not permit checkpoint cleanup") + } +} + +func TestDisabledRegistryKeepsArchivedClaimCheckpoint(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + mustArchive(t, m, sb) + ck, epoch := sb.ArchiveCk, sb.ArchiveEpoch + drainClaimRow(t, c, sb.ID) + restarted := newManagerWithoutRegistry(t, m, c) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + restored, ok := restarted.byID(sb.ID) + if !ok || restored.ArchiveCk != ck || restored.ArchiveEpoch != epoch || !ckExists(t, restarted, ck) || restarted.deleteMarked(ck) { + t.Fatal("replica retirement changed the archived claim or its checkpoint") + } +} + func TestReplicaFailedClearQueuesCheckpointCleanup(t *testing.T) { m, _ := newReplicaVolumeManager(t) sb := mustClaim(t, m, testKey) @@ -91,3 +166,32 @@ func TestReplicaFailedClearQueuesCheckpointCleanup(t *testing.T) { t.Fatal("checkpoint cleanup did not finish after its reference retired") } } + +func newManagerWithoutRegistry(t *testing.T, old *Manager, c *failoverCluster) *Manager { + t.Helper() + cfg := &config.Config{ + APIToken: "root", DataDir: old.dataDir, CheckpointDir: c.ckDir, + Pools: []config.PoolSpec{{PoolKey: testKey}}, Volumes: old.cfg.Volumes, + } + m, err := NewManager(t.Context(), cfg, old.eng, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + if m.archives != nil { + t.Fatal("failover-disabled fixture unexpectedly has a registry") + } + return m +} + +func drainClaimRow(t *testing.T, c *failoverCluster, id string) { + t.Helper() + rec, ok, err := c.reg.Get(t.Context(), id) + if err != nil { + t.Fatal(err) + } + if ok { + if err = c.reg.Delete(t.Context(), id, rec.Epoch); err != nil { + t.Fatal(err) + } + } +} diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index e36dee45..5f0f1a66 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -424,7 +424,7 @@ func TestReplicaLeftoverJournalWithFailoverOffStillServesStreams(t *testing.T) { m := newTestManagerAt(t, eng, dir, config.PoolSpec{PoolKey: testKey, Warm: 1}) sb := mustClaim(t, m, testKey) m.mu.Lock() - sb.ReplicaCk = "ck_leftover" + sb.ReplicaCk = "ck_0123456789abcdef" js := m.store.set(sb) m.mu.Unlock() if err := m.store.commit(js); err != nil { @@ -436,8 +436,11 @@ func TestReplicaLeftoverJournalWithFailoverOffStillServesStreams(t *testing.T) { t.Fatalf("reconcile: %v", err) } held, ok := restarted.byID(sb.ID) - if !ok || held.ReplicaCk != "ck_leftover" || held.Replicated() { - t.Fatalf("claim held %v replica %q replicated %v, want the leftover kept and not replicated", ok, held.ReplicaCk, held.Replicated()) + if !ok { + t.Fatal("reconcile lost the claim while retiring its leftover replica") + } + if held.HasReplica() || held.Replicated() { + t.Fatalf("claim held %v replica %q replicated %v, want the leftover retired and claim retained", ok, held.ReplicaCk, held.Replicated()) } _, fenced, done, err := restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil || fenced != nil { diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index 3c747fca..f8f893ec 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -99,8 +99,8 @@ func TestReplicaPendingCheckpointCleanupWaitsForAcknowledgment(t *testing.T) { if sb.ReplicaPending != "" || m.store.synced() { t.Fatal("expected memory clear before snapshot acknowledgment") } - if err := m.deleteArchiveCk(t.Context(), pending); err == nil { - t.Fatal("orphan delete ignored unacknowledged claim") + if deleted, err := m.deleteArchiveCk(t.Context(), pending); err != nil || deleted { + t.Fatalf("orphan delete ignored unacknowledged claim: deleted=%t, %v", deleted, err) } m.retryArchiveDeletes(t.Context()) if !ckExists(t, m, pending) || !m.deleteMarked(pending) { diff --git a/sandboxd/pool/volume_replication_policy_test.go b/sandboxd/pool/volume_replication_policy_test.go index e89873ae..f14c421f 100644 --- a/sandboxd/pool/volume_replication_policy_test.go +++ b/sandboxd/pool/volume_replication_policy_test.go @@ -101,8 +101,8 @@ func TestArchiveCheckpointCleanupIgnoresUnrelatedClaimWrites(t *testing.T) { if m.store.synced() { t.Fatal("expected an unrelated unacknowledged claim write") } - if err := m.deleteArchiveCk(t.Context(), ck); err != nil { - t.Fatalf("unrelated claim delayed checkpoint cleanup: %v", err) + if deleted, err := m.deleteArchiveCk(t.Context(), ck); err != nil || !deleted { + t.Fatalf("unrelated claim delayed checkpoint cleanup: deleted=%t, %v", deleted, err) } if ckExists(t, m, ck) || m.deleteMarked(ck) { t.Fatal("unreferenced checkpoint or delete marker remained") From c576b32cd7efad944801700d6a4a79cf74dd0429 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 08:04:18 +0000 Subject: [PATCH 59/72] pool: remove unpublished writable intent compatibility --- docs/sandboxd-api.md | 15 ++-- docs/volume-controller.md | 6 +- sandboxd/pool/archive.go | 1 - sandboxd/pool/reconcile.go | 1 - sandboxd/pool/volume_mutation.go | 21 +---- sandboxd/pool/volume_replica_test.go | 10 +-- .../pool/volume_replication_policy_test.go | 79 +++++-------------- sandboxd/server/server.go | 2 +- sandboxd/server/volume_mutation_test.go | 11 ++- 9 files changed, 47 insertions(+), 99 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 6f8949cb..42c274ea 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -541,19 +541,18 @@ remain available. The effective pool is the claim's policy source when it has one. Changing `replicate_seconds` requires a daemon restart. A non-replicating pool can still have an existing or pending replica after a -restart. Live RW additions return 409 until that claim's replicas and recovery +restart. Live RW additions return 503 with `Retry-After` until that claim's replicas and recovery checkpoint references have retired. The background scan retires existing replicas; keep the controller's reservation and access gate closed, then retry -the request. A failed claim write retains the affected checkpoint reference +the request. When disabling `archive_failover`, follow +[the registry transition requirements](deploy.md#disabling-archive-failover); +startup without a registry retires only local replica references, not remote +claim rows. A failed claim write retains the affected checkpoint reference until a successful snapshot acknowledges its removal. Unrelated claims' writes and checkpoint references do not refuse this attachment. -Older versions may have already recorded a desired RW target while replication -was enabled. These targets remain visible after restart and retain their holds. -They cannot attach while the pool replicates, but callers can remove them or -release the claim. Cleanup of applied volumes runs before a remaining pending RW -is refused, so a remove request can complete an RO detach and still return a -partial 409 for that pending writer. Inspect the returned view before retrying. +A retry of an admitted desired volume completes that operation without creating +a new admission. Its volume hold continues to prevent replica publication. After a request ends, `converging` is false unless ordinary recovery-reference retirement remains active; no background worker attaches the desired disk. diff --git a/docs/volume-controller.md b/docs/volume-controller.md index cf0633ad..48259d56 100644 --- a/docs/volume-controller.md +++ b/docs/volume-controller.md @@ -18,8 +18,10 @@ preserve this sleep workflow when archiving is enabled. Live RW additions require `replicate_seconds: 0`; a replicating pool returns 409 without recording the new target or reserving the volume. Changing this setting requires a daemon restart. A non-replicating pool may still hold an old -replica or recovery checkpoint reference. Keep the reservation and access gate -closed and retry attachment after that claim's retirement has been acknowledged. +replica or recovery checkpoint reference, which returns 503 with `Retry-After`. +Keep the reservation and access gate closed and retry attachment after that +claim's retirement has been acknowledged. When disabling failover, follow +[the registry transition requirements](deploy.md#disabling-archive-failover). A fixed delay does not acknowledge retirement. See [the live-volume retirement contract](sandboxd-api.md#partial-failure-and-recovery). diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 3bc7f2e4..e6380456 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -351,7 +351,6 @@ func (m *Manager) clearArchiveCk(ckID string) error { return err } -// deleteArchiveCk queues deletion and reports whether references allowed it to finish. func (m *Manager) deleteArchiveCk(ctx context.Context, ckID string) (bool, error) { if err := m.markArchiveCk(ckID); err != nil { return false, fmt.Errorf("track: %w", err) diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index d4072fac..43fed333 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -144,7 +144,6 @@ func (m *Manager) retireReplicasWithoutRegistry(claims map[string]*types.Sandbox } } // Without a registry this manager has no node identity for later orphan scans. - // Queue cleanup before the last persisted replica reference can disappear. for id, err := range m.markArchiveCks(checkpoints) { return fmt.Errorf("queue replica without registry %s: %w", id, err) } diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 43a46a1e..d73af5bd 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -151,7 +151,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v m.mu.Unlock() for _, v := range pending { failedName = v.Name - if err = m.checkVolumeApply(ctx, sb, run, v); err != nil { + if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } repair, resolveErr := m.resolveCatalogVolume(ctx, v) @@ -326,18 +326,6 @@ func (m *Manager) checkLiveAdditions(sb *types.Sandbox, additions []types.Volume return nil } -func (m *Manager) checkVolumeApply(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume) error { - m.mu.Lock() - defer m.mu.Unlock() - if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { - return err - } - if v.RW() { - return m.liveWriterError(sb) - } - return nil -} - // liveWriterError checks the policy and recoverable replicas of this claim; callers hold m.mu. func (m *Manager) liveWriterError(sb *types.Sandbox) error { if m.view.Load().poolReplicate[sb.PolicyKey()] > 0 { @@ -442,11 +430,6 @@ func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { state = types.VolumePending } message := m.volumeErrors[sb][v.Name] - if state == types.VolumePending && v.RW() { - if err := m.liveWriterError(sb); err != nil { - message = err.Error() - } - } if message == "" && state != types.VolumeAttached && !view.Converging { message = volumeChangeInterrupted } @@ -466,7 +449,7 @@ func (m *Manager) recordVolumeError(sb *types.Sandbox, run *volumeRun, name stri return } errs := m.volumeErrors[sb] - if name != "" && !errors.Is(err, ErrReplicaPresent) && !errors.Is(err, ErrReplicatingPool) { + if name != "" { message := volumeChangeFailed if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { message = volumeChangeInterrupted diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index f8f893ec..e723753a 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -159,14 +159,12 @@ func TestReplicaSettledCheckpointCleanupRetriesAfterFailedCommit(t *testing.T) { } } -func TestReplicaRecoveryDoesNotRepublishVolumeIntent(t *testing.T) { +func TestReplicaRecoveryDoesNotRepublishReadOnlyHold(t *testing.T) { m, c := newReplicaVolumeManager(t) sb := mustClaim(t, m, testKey) replicateNow(t, m, sb) - // Reconstruct the durable state a pending writer leaves before replica retirement. - sb.VolumesWant = []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - if err := m.store.commit(m.store.set(sb)); err != nil { - t.Fatal(err) + if view, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil || len(view.Volumes) != 1 { + t.Fatalf("read-only live attachment: %+v, %v", view, err) } if _, err := c.reg.Consume(t.Context(), sb.ID, sb.ArchiveEpoch); err != nil { t.Fatal(err) @@ -177,7 +175,7 @@ func TestReplicaRecoveryDoesNotRepublishVolumeIntent(t *testing.T) { } m.reconcileOwnRow(t.Context(), rec, time.Now()) if rec, ok, err = c.reg.Get(t.Context(), sb.ID); err != nil || !ok || !rec.Consumed { - t.Fatalf("recovery republished desired writer: %+v, %t, %v", rec, ok, err) + t.Fatalf("recovery republished read-only hold: %+v, %t, %v", rec, ok, err) } } diff --git a/sandboxd/pool/volume_replication_policy_test.go b/sandboxd/pool/volume_replication_policy_test.go index f14c421f..d2a66f44 100644 --- a/sandboxd/pool/volume_replication_policy_test.go +++ b/sandboxd/pool/volume_replication_policy_test.go @@ -212,7 +212,7 @@ func TestLiveWriterWaitsOnlyForItsOwnCheckpointReferences(t *testing.T) { } } -func TestLegacyWriterIntentCanRemoveReadOnlyVolumes(t *testing.T) { +func TestAdmittedWriterRetryAfterRestartCanRemoveReader(t *testing.T) { eng := newFakeEngine() catalog := liveCatalog(t) m := newVolumeManager(t, eng, catalog) @@ -220,20 +220,24 @@ func TestLegacyWriterIntentCanRemoveReadOnlyVolumes(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } - m.mu.Lock() - sb.VolumesWant = append(slices.Clone(sb.Volumes), types.Volume{Name: "other", Mode: types.VolumeModeRW, AttachOnly: true}) - snap := m.store.set(sb) - m.mu.Unlock() - if err := m.store.commit(snap); err != nil { - t.Fatal(err) + eng.liveVolumeHook = func(op string) error { + if op == "attach" { + return errors.New("injected writable attachment failure") + } + return nil + } + writer := types.Volume{Name: "other", Mode: types.VolumeModeRW, AttachOnly: true} + view, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{writer}}) + if err == nil || len(view.VolumesWant) != 2 || len(view.Volumes) != 1 { + t.Fatalf("attachment failure did not retain admitted target: %+v, %v", view, err) } + eng.liveVolumeHook = nil c := newReplicaCluster(t) spec := archivePool(keepAnHour) spec.ReplicateSeconds = c.replicate cfg := &config.Config{APIToken: "root", DataDir: m.dataDir, CheckpointDir: c.ckDir, Volumes: catalog, Pools: []config.PoolSpec{spec}, Mesh: &config.MeshConfig{NodeID: "a"}} backends := testBackends(t, cfg) backends.Archives = c.reg - var err error m, err = NewManager(t.Context(), cfg, eng, testSecrets(t), backends) if err != nil { t.Fatal(err) @@ -243,58 +247,13 @@ func TestLegacyWriterIntentCanRemoveReadOnlyVolumes(t *testing.T) { t.Fatal(err) } m.liftFence() - got, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Remove: []string{"data"}}) - if !errors.Is(err, ErrReplicatingPool) || len(got.Volumes) != 0 || len(got.VolumesWant) != 1 || got.VolumesWant[0].Name != "other" { - t.Fatalf("legacy writer blocked reader cleanup: %+v, %v", got, err) - } - if got, err = m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil || len(got.VolumesWant) != 0 || len(got.Volumes) != 0 { - t.Fatalf("legacy writer cannot be canceled: %+v, %v", got, err) - } - if volumeHoldersOf(m, "other") != (volumeHolders{}) { - t.Fatal("canceled legacy writer retained admission") - } -} - -func TestLegacyWriterApplyWaitsForItsRetirementReference(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - replicateNow(t, m, sb) - writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - m.mu.Lock() - sb.VolumesWant = writer - snap := m.store.set(sb) - m.mu.Unlock() - if err := m.store.commit(snap); err != nil { - t.Fatal(err) - } - m = restartReplicaVolumeManager(t, m, c) - sb, _ = m.byID(sb.ID) - ctx, run, err := m.beginVolumeRun(t.Context(), sb, "") - if err != nil { - t.Fatal(err) - } - defer m.finishVolumeRun(sb, run) - if err = m.prepareVolumeTarget(ctx, sb, run, writer, nil, nil); err != nil { - t.Fatal(err) + view, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Remove: []string{"data"}}) + if err != nil || !slices.Equal(view.VolumesWant, []types.Volume{writer}) || !slices.Equal(view.Volumes, []types.Volume{writer}) { + t.Fatalf("admitted writer retry prevented reader removal: %+v, %v", view, err) } - before := m.eng.(*fakeEngine).volumeOpsLog() - m.store.recommitting.Store(true) - m.store.replaceFile = func([]byte) error { return errors.New("injected legacy retirement failure") } - m.retireReplica(t.Context(), sb) - if sb.HasReplica() || len(m.store.checkpointRefs(sb.ID)) == 0 { - t.Fatal("expected only an unacknowledged retirement reference") - } - if err = m.checkVolumeApply(ctx, sb, run, writer[0]); !errors.Is(err, ErrReplicaPresent) { - t.Fatalf("legacy apply ignored its retirement reference: %v", err) - } - if !slices.Equal(before, m.eng.(*fakeEngine).volumeOpsLog()) { - t.Fatal("legacy writer called the device before retirement acknowledgement") - } - m.store.replaceFile = nil - if err = m.store.commit(m.store.mark()); err != nil { - t.Fatal(err) - } - if err = m.checkVolumeApply(ctx, sb, run, writer[0]); err != nil { - t.Fatalf("acknowledged legacy retirement blocked apply: %v", err) + m.replicateOnce(t.Context()) + waitFor(t, func() bool { return !m.replicating.Load() }) + if rec, ok, err := c.reg.Get(t.Context(), sb.ID); err != nil || ok { + t.Fatalf("admitted writer acquired a replica row: %+v, exists=%t, %v", rec, ok, err) } } diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 57201f86..44c223cb 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -67,7 +67,7 @@ var poolErrHTTP = []struct { {pool.ErrNoEgressFork, http.StatusConflict, ""}, {pool.ErrVolumeCapture, http.StatusConflict, ""}, {pool.ErrVolumeBusy, http.StatusConflict, ""}, - {pool.ErrReplicaPresent, http.StatusConflict, ""}, + {pool.ErrReplicaPresent, http.StatusServiceUnavailable, ""}, {pool.ErrReplicatingPool, http.StatusConflict, ""}, {pool.ErrVolumeNeedsRecovery, http.StatusConflict, ""}, {pool.ErrArchived, http.StatusConflict, ""}, diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 0c3cd4be..626188a5 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -165,7 +165,16 @@ func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatal(err) } - if resp.StatusCode != http.StatusConflict || !strings.HasPrefix(got.Error, refusal.Error()) || len(got.States) != 1 || got.States[0].Error != cause { + wantStatus := http.StatusConflict + if errors.Is(refusal, pool.ErrReplicaPresent) { + wantStatus = http.StatusServiceUnavailable + if resp.Header.Get("Retry-After") == "" { + t.Fatal("temporary replica retirement refusal has no Retry-After") + } + } else if resp.Header.Get("Retry-After") != "" { + t.Fatal("permanent conflict advertises a retry delay") + } + if resp.StatusCode != wantStatus || !strings.HasPrefix(got.Error, refusal.Error()) || len(got.States) != 1 || got.States[0].Error != cause { t.Fatalf("status=%d view=%+v", resp.StatusCode, got) } }) From f2f2404d3c69fd494783dea2c027ac7ba6535803 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 08:08:06 +0000 Subject: [PATCH 60/72] pool: cover archive-intent recovery without a registry --- sandboxd/pool/replica_cleanup_test.go | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/sandboxd/pool/replica_cleanup_test.go b/sandboxd/pool/replica_cleanup_test.go index 753db22e..2c12b7c0 100644 --- a/sandboxd/pool/replica_cleanup_test.go +++ b/sandboxd/pool/replica_cleanup_test.go @@ -137,6 +137,38 @@ func TestDisabledRegistryKeepsArchivedClaimCheckpoint(t *testing.T) { } } +func TestDisabledRegistryKeepsArchiveIntentCheckpoint(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + replica, epoch := sb.ReplicaCk, sb.ArchiveEpoch + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + m.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := m.archive(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatalf("archive intent fixture: %v", err) + } + pending := sb.ArchivePending + if pending == "" || sb.ArchiveCk != "" || sb.ReplicaCk != replica || sb.ArchiveEpoch != epoch { + t.Fatal("failed archive did not retain its checkpoint intent and replica") + } + drainClaimRow(t, c, sb.ID) + restarted := newManagerWithoutRegistry(t, m, c) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + restored, ok := restarted.byID(sb.ID) + if !ok || restored.ArchivePending != pending || restored.ReplicaCk != replica || restored.ArchiveEpoch != epoch { + t.Fatal("replica retirement changed an in-progress archive") + } + for _, ck := range []string{pending, replica} { + if !ckExists(t, restarted, ck) || restarted.deleteMarked(ck) { + t.Fatal("replica retirement queued or deleted an archive dependency") + } + } +} + func TestReplicaFailedClearQueuesCheckpointCleanup(t *testing.T) { m, _ := newReplicaVolumeManager(t) sb := mustClaim(t, m, testKey) From 9c660508ef0d54830a45a9bc1b47bf394d889d17 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 08:40:00 +0000 Subject: [PATCH 61/72] pool: reproduce replica retirement without a registry --- sandboxd/pool/pool.go | 1 + sandboxd/pool/replica_cleanup_test.go | 36 +++++++++++++++++++++++++++ sandboxd/server/server.go | 1 + 3 files changed, 38 insertions(+) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index baaaa864..9e4ddb0d 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -105,6 +105,7 @@ var ( ErrReplicaPresent = errors.New("live writable attachment requires replica and recovery checkpoint retirement") ErrReplicatingPool = errors.New("live writable volumes require a non-replicating pool") + ErrReplicaStranded = errors.New("claim keeps a replica from a failover configuration; release it or re-enable archive_failover") // Replaying a journal takes a writable mount, so readers stay out. ErrVolumeNeedsRecovery = errors.New("volume needs recovery by a writable claim") diff --git a/sandboxd/pool/replica_cleanup_test.go b/sandboxd/pool/replica_cleanup_test.go index 2c12b7c0..712d97cb 100644 --- a/sandboxd/pool/replica_cleanup_test.go +++ b/sandboxd/pool/replica_cleanup_test.go @@ -54,6 +54,42 @@ func TestReconcileRetiresReplicasWhenFailoverDisabled(t *testing.T) { } } +func TestFailoverDisabledKeepsReplicaAndRefusesLiveWriter(t *testing.T) { + for _, pending := range []string{"published", "pending"} { + t.Run(pending, func(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + if pending == "pending" { + m.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + } else { + replicateNow(t, m, sb) + } + replica, intent := sb.ReplicaCk, sb.ReplicaPending + restarted := newManagerWithoutRegistry(t, m, c) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + restored, ok := restarted.byID(sb.ID) + if !ok || restored.ReplicaCk != replica || restored.ReplicaPending != intent { + t.Fatalf("startup without a registry changed replica fields: ok=%t replica=%q pending=%q", ok, restored.ReplicaCk, restored.ReplicaPending) + } + for _, ck := range []string{replica, intent} { + if ck != "" && (!ckExists(t, restarted, ck) || restarted.deleteMarked(ck)) { + t.Fatalf("startup without a registry queued or deleted checkpoint %s", ck) + } + } + want := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + view, err := restarted.SetVolumes(t.Context(), sb.ID, "", want) + if !errors.Is(err, ErrReplicaStranded) || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("live writer on a stranded replica: err=%v volumes=%v want=%v", err, view.Volumes, view.VolumesWant) + } + }) + } +} + func TestDisabledReplicaCleanupSurvivesFailedStartupRewrite(t *testing.T) { for _, renamed := range []bool{false, true} { t.Run(fmt.Sprintf("renamed=%t", renamed), func(t *testing.T) { diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 44c223cb..13fba1d5 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -69,6 +69,7 @@ var poolErrHTTP = []struct { {pool.ErrVolumeBusy, http.StatusConflict, ""}, {pool.ErrReplicaPresent, http.StatusServiceUnavailable, ""}, {pool.ErrReplicatingPool, http.StatusConflict, ""}, + {pool.ErrReplicaStranded, http.StatusConflict, ""}, {pool.ErrVolumeNeedsRecovery, http.StatusConflict, ""}, {pool.ErrArchived, http.StatusConflict, ""}, {pool.ErrPaused, http.StatusConflict, ""}, From b181d4f191162471d53d7e123ea8f386c3967ab7 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 08:43:01 +0000 Subject: [PATCH 62/72] pool: keep stranded replicas when failover is disabled Startup without a registry used to clear replica fields and queue their checkpoints for deletion. A failed rewrite followed by a rollback to the registry could then delete a checkpoint a peer had adopted, and the procedure depended on operator preconditions nothing could observe. Without a registry nothing can retire a replica, so live writable additions on such a claim now return a permanent 409 instead, and its checkpoint and remote row stay intact. The expected 503 while a replica retires no longer logs at error level. --- docs/deploy.md | 19 +-- docs/sandboxd-api.md | 18 +-- docs/volume-controller.md | 5 +- sandboxd/pool/reconcile.go | 25 ---- sandboxd/pool/replica_cleanup_test.go | 174 ------------------------ sandboxd/pool/replica_test.go | 9 +- sandboxd/pool/volume_mutation.go | 4 + sandboxd/server/volume_mutation.go | 2 +- sandboxd/server/volume_mutation_test.go | 2 +- 9 files changed, 27 insertions(+), 231 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 566e1bc7..64352ecf 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -169,19 +169,12 @@ fragment the warm pools): ### Disabling archive failover -Set `replicate_seconds: 0` while `archive_failover` remains enabled, restart, -and let the existing replicas and their recovery references retire before -disabling failover. Keep the old registry reachable during that transition. -Before removing it, drain its remaining claim rows. If adopters must be stopped -first, keep every adopter shut down or disabled until those rows are drained or -the adoption domain is permanently retired. Disabling the local registry does -not consume remote rows or fence another node. - -On startup without a registry, sandboxd clears replica fields from local running -or sleeping claims and queues their checkpoints for deletion before rewriting -the claims file. A failed rewrite keeps the affected checkpoints protected until -their references retire. Archived claims and unfinished archive transitions -retain their archive fields and checkpoints. +Set `replicate_seconds: 0` while `archive_failover` is still enabled, then +restart, so the replica scan retires existing replicas. A claim that still has +a replica when `archive_failover` is disabled keeps it. sandboxd does not +delete that checkpoint or touch the remote registry row, and live writable +volume additions on the claim return 409 until it is released or failover is +re-enabled. ### Reading sleeping claims without waking them diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 42c274ea..8fa58315 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -541,15 +541,15 @@ remain available. The effective pool is the claim's policy source when it has one. Changing `replicate_seconds` requires a daemon restart. A non-replicating pool can still have an existing or pending replica after a -restart. Live RW additions return 503 with `Retry-After` until that claim's replicas and recovery -checkpoint references have retired. The background scan retires existing -replicas; keep the controller's reservation and access gate closed, then retry -the request. When disabling `archive_failover`, follow -[the registry transition requirements](deploy.md#disabling-archive-failover); -startup without a registry retires only local replica references, not remote -claim rows. A failed claim write retains the affected checkpoint reference -until a successful snapshot acknowledges its removal. Unrelated claims' writes -and checkpoint references do not refuse this attachment. +restart. Live RW additions return 503 with `Retry-After` until that claim's +replicas and recovery checkpoint references have retired. The background scan +retires existing replicas; keep the controller's reservation and access gate +closed, then retry the request. Without `archive_failover` nothing retires a +leftover replica, so live RW additions on that claim return 409 instead; see +[disabling archive failover](deploy.md#disabling-archive-failover). A failed +claim write retains the affected checkpoint reference until a successful +snapshot acknowledges its removal. Unrelated claims' writes and checkpoint +references do not refuse this attachment. A retry of an admitted desired volume completes that operation without creating a new admission. Its volume hold continues to prevent replica publication. diff --git a/docs/volume-controller.md b/docs/volume-controller.md index 48259d56..ee9e3f9f 100644 --- a/docs/volume-controller.md +++ b/docs/volume-controller.md @@ -20,8 +20,9 @@ Live RW additions require `replicate_seconds: 0`; a replicating pool returns setting requires a daemon restart. A non-replicating pool may still hold an old replica or recovery checkpoint reference, which returns 503 with `Retry-After`. Keep the reservation and access gate closed and retry attachment after that -claim's retirement has been acknowledged. When disabling failover, follow -[the registry transition requirements](deploy.md#disabling-archive-failover). +claim's retirement has been acknowledged. A claim that keeps a replica after +[archive failover is disabled](deploy.md#disabling-archive-failover) returns 409 +and must be released. A fixed delay does not acknowledge retirement. See [the live-volume retirement contract](sandboxd-api.md#partial-failure-and-recovery). diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 43fed333..e4992bca 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -24,9 +24,6 @@ func (m *Manager) Reconcile(ctx context.Context) error { if err != nil { return err } - if err = m.retireReplicasWithoutRegistry(claims); err != nil { - return err - } fenced := m.fenceArchived(ctx, claims) vms, err := m.eng.List(ctx) if err != nil { @@ -133,28 +130,6 @@ func (m *Manager) Reconcile(ctx context.Context) error { return saveErr } -func (m *Manager) retireReplicasWithoutRegistry(claims map[string]*types.Sandbox) error { - if m.archives != nil { - return nil - } - var checkpoints []string - for _, sb := range claims { - if sb.ArchiveCk == "" && sb.ArchivePending == "" { - checkpoints = append(checkpoints, checkpointIDs(sb)...) - } - } - // Without a registry this manager has no node identity for later orphan scans. - for id, err := range m.markArchiveCks(checkpoints) { - return fmt.Errorf("queue replica without registry %s: %w", id, err) - } - for _, sb := range claims { - if sb.ArchiveCk == "" && sb.ArchivePending == "" { - sb.ReplicaCk, sb.ReplicaPending, sb.ArchiveEpoch = "", "", 0 - } - } - return nil -} - // sweepStaleVMs removes sbx-prefixed VMs no claim owns and returns the ones confirmed gone. func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRecord, owned map[string]bool) map[string]bool { var stale []string diff --git a/sandboxd/pool/replica_cleanup_test.go b/sandboxd/pool/replica_cleanup_test.go index 712d97cb..cbeef1b2 100644 --- a/sandboxd/pool/replica_cleanup_test.go +++ b/sandboxd/pool/replica_cleanup_test.go @@ -2,58 +2,12 @@ package pool import ( "errors" - "fmt" - "os" - "path/filepath" "testing" "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestReconcileRetiresReplicasWhenFailoverDisabled(t *testing.T) { - for _, pending := range []bool{false, true} { - t.Run(fmt.Sprintf("pending=%t", pending), func(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - if pending { - m.archives = &hookRegistry{Registry: c.reg, failPut: true} - if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { - t.Fatal(err) - } - } else { - replicateNow(t, m, sb) - } - ck := sb.ReplicaCk - if pending { - ck = sb.ReplicaPending - } - drainClaimRow(t, c, sb.ID) - restarted := newManagerWithoutRegistry(t, m, c) - if err := restarted.Reconcile(t.Context()); err != nil { - t.Fatal(err) - } - restored, ok := restarted.byID(sb.ID) - if !ok { - t.Fatal("disabling failover lost the running claim") - } - if restored.HasReplica() || restored.ArchiveEpoch != 0 { - t.Errorf("disabled failover retained replica fields: replica=%q pending=%q epoch=%d", restored.ReplicaCk, restored.ReplicaPending, restored.ArchiveEpoch) - } - want := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - for range 3 { - restarted.replicateOnce(t.Context()) - } - if view, applyErr := restarted.SetVolumes(t.Context(), sb.ID, "", want); applyErr != nil || len(view.Volumes) != 1 { - t.Errorf("disabled failover left live writer permanently blocked: %+v, %v", view, applyErr) - } - if ckExists(t, restarted, ck) && !restarted.deleteMarked(ck) { - t.Error("disabled failover left the retired checkpoint without a cleanup marker") - } - }) - } -} - func TestFailoverDisabledKeepsReplicaAndRefusesLiveWriter(t *testing.T) { for _, pending := range []string{"published", "pending"} { t.Run(pending, func(t *testing.T) { @@ -90,121 +44,6 @@ func TestFailoverDisabledKeepsReplicaAndRefusesLiveWriter(t *testing.T) { } } -func TestDisabledReplicaCleanupSurvivesFailedStartupRewrite(t *testing.T) { - for _, renamed := range []bool{false, true} { - t.Run(fmt.Sprintf("renamed=%t", renamed), func(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - replicateNow(t, m, sb) - ck := sb.ReplicaCk - drainClaimRow(t, c, sb.ID) - restarted := newManagerWithoutRegistry(t, m, c) - restarted.store.replaceFile = func(raw []byte) error { - if renamed { - if err := restarted.store.replace(raw); err != nil { - return err - } - } - return errors.New("injected startup replacement failure") - } - if err := restarted.Reconcile(t.Context()); err == nil { - t.Fatal("startup replacement unexpectedly succeeded") - } - if !restarted.deleteMarked(ck) || !ckExists(t, restarted, ck) { - t.Fatal("failed startup lost a checkpoint or its cleanup intent") - } - persisted, err := newClaimStore(m.dataDir, false).load() - if err != nil || persisted[sb.ID] == nil || persisted[sb.ID].HasReplica() == renamed { - t.Fatalf("startup failure did not reach the expected persisted state: %+v, %v", persisted, err) - } - recovered := newManagerWithoutRegistry(t, restarted, c) - if err = recovered.Reconcile(t.Context()); err != nil { - t.Fatal(err) - } - if restored, ok := recovered.byID(sb.ID); !ok || restored.HasReplica() || ckExists(t, recovered, ck) || recovered.deleteMarked(ck) { - t.Fatal("next startup did not retain the claim and finish queued cleanup") - } - }) - } -} - -func TestDisabledReplicaCleanupRequiresDurableMarker(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - replicateNow(t, m, sb) - ck := sb.ReplicaCk - drainClaimRow(t, c, sb.ID) - markerDir := filepath.Join(m.dataDir, archiveDeleteDir) - if err := os.WriteFile(markerDir, []byte("blocked"), 0o600); err != nil { - t.Fatal(err) - } - restarted := newManagerWithoutRegistry(t, m, c) - if err := restarted.Reconcile(t.Context()); err == nil { - t.Fatal("startup retired a replica without a durable cleanup marker") - } - persisted, err := newClaimStore(m.dataDir, false).load() - if err != nil || persisted[sb.ID] == nil || persisted[sb.ID].ReplicaCk != ck || !ckExists(t, restarted, ck) { - t.Fatalf("failed marker write changed the recoverable replica: %+v, %v", persisted, err) - } - if err = os.Remove(markerDir); err != nil { - t.Fatal(err) - } - if err = restarted.Reconcile(t.Context()); err != nil { - t.Fatal(err) - } - if ckExists(t, restarted, ck) { - t.Fatal("marker repair did not permit checkpoint cleanup") - } -} - -func TestDisabledRegistryKeepsArchivedClaimCheckpoint(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - mustArchive(t, m, sb) - ck, epoch := sb.ArchiveCk, sb.ArchiveEpoch - drainClaimRow(t, c, sb.ID) - restarted := newManagerWithoutRegistry(t, m, c) - if err := restarted.Reconcile(t.Context()); err != nil { - t.Fatal(err) - } - restored, ok := restarted.byID(sb.ID) - if !ok || restored.ArchiveCk != ck || restored.ArchiveEpoch != epoch || !ckExists(t, restarted, ck) || restarted.deleteMarked(ck) { - t.Fatal("replica retirement changed the archived claim or its checkpoint") - } -} - -func TestDisabledRegistryKeepsArchiveIntentCheckpoint(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - replicateNow(t, m, sb) - replica, epoch := sb.ReplicaCk, sb.ArchiveEpoch - if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { - t.Fatal(err) - } - m.archives = &hookRegistry{Registry: c.reg, failPut: true} - if err := m.archive(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { - t.Fatalf("archive intent fixture: %v", err) - } - pending := sb.ArchivePending - if pending == "" || sb.ArchiveCk != "" || sb.ReplicaCk != replica || sb.ArchiveEpoch != epoch { - t.Fatal("failed archive did not retain its checkpoint intent and replica") - } - drainClaimRow(t, c, sb.ID) - restarted := newManagerWithoutRegistry(t, m, c) - if err := restarted.Reconcile(t.Context()); err != nil { - t.Fatal(err) - } - restored, ok := restarted.byID(sb.ID) - if !ok || restored.ArchivePending != pending || restored.ReplicaCk != replica || restored.ArchiveEpoch != epoch { - t.Fatal("replica retirement changed an in-progress archive") - } - for _, ck := range []string{pending, replica} { - if !ckExists(t, restarted, ck) || restarted.deleteMarked(ck) { - t.Fatal("replica retirement queued or deleted an archive dependency") - } - } -} - func TestReplicaFailedClearQueuesCheckpointCleanup(t *testing.T) { m, _ := newReplicaVolumeManager(t) sb := mustClaim(t, m, testKey) @@ -250,16 +89,3 @@ func newManagerWithoutRegistry(t *testing.T, old *Manager, c *failoverCluster) * } return m } - -func drainClaimRow(t *testing.T, c *failoverCluster, id string) { - t.Helper() - rec, ok, err := c.reg.Get(t.Context(), id) - if err != nil { - t.Fatal(err) - } - if ok { - if err = c.reg.Delete(t.Context(), id, rec.Epoch); err != nil { - t.Fatal(err) - } - } -} diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 5f0f1a66..e36dee45 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -424,7 +424,7 @@ func TestReplicaLeftoverJournalWithFailoverOffStillServesStreams(t *testing.T) { m := newTestManagerAt(t, eng, dir, config.PoolSpec{PoolKey: testKey, Warm: 1}) sb := mustClaim(t, m, testKey) m.mu.Lock() - sb.ReplicaCk = "ck_0123456789abcdef" + sb.ReplicaCk = "ck_leftover" js := m.store.set(sb) m.mu.Unlock() if err := m.store.commit(js); err != nil { @@ -436,11 +436,8 @@ func TestReplicaLeftoverJournalWithFailoverOffStillServesStreams(t *testing.T) { t.Fatalf("reconcile: %v", err) } held, ok := restarted.byID(sb.ID) - if !ok { - t.Fatal("reconcile lost the claim while retiring its leftover replica") - } - if held.HasReplica() || held.Replicated() { - t.Fatalf("claim held %v replica %q replicated %v, want the leftover retired and claim retained", ok, held.ReplicaCk, held.Replicated()) + if !ok || held.ReplicaCk != "ck_leftover" || held.Replicated() { + t.Fatalf("claim held %v replica %q replicated %v, want the leftover kept and not replicated", ok, held.ReplicaCk, held.Replicated()) } _, fenced, done, err := restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil || fenced != nil { diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index d73af5bd..1c59db9a 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -331,6 +331,10 @@ func (m *Manager) liveWriterError(sb *types.Sandbox) error { if m.view.Load().poolReplicate[sb.PolicyKey()] > 0 { return ErrReplicatingPool } + if sb.HasReplica() && m.archives == nil { + // only the replica scan retires a replica, and it needs the registry + return ErrReplicaStranded + } if sb.HasReplica() || len(m.store.checkpointRefs(sb.ID)) > 0 { return ErrReplicaPresent } diff --git a/sandboxd/server/volume_mutation.go b/sandboxd/server/volume_mutation.go index 1e4f86f9..86dd6398 100644 --- a/sandboxd/server/volume_mutation.go +++ b/sandboxd/server/volume_mutation.go @@ -60,7 +60,7 @@ func (s *Server) handleSandboxVolumes(w http.ResponseWriter, r *http.Request) { code, view.Error = mapped, msg } } - if code >= http.StatusInternalServerError { + if code >= http.StatusInternalServerError && !errors.Is(err, pool.ErrReplicaPresent) { log.WithFunc("server.handleSandboxVolumes").Errorf(r.Context(), err, "converge volumes for %s", id) } } diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 626188a5..2c593b41 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -153,7 +153,7 @@ func TestLiveVolumeRoutesHideOtherTenants(t *testing.T) { func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { const cause = "cannot unmount /data; close guest references and retry" - for _, refusal := range []error{pool.ErrVolumeBusy, pool.ErrVolumeCapture, pool.ErrReplicaPresent, pool.ErrReplicatingPool} { + for _, refusal := range []error{pool.ErrVolumeBusy, pool.ErrVolumeCapture, pool.ErrReplicaPresent, pool.ErrReplicatingPool, pool.ErrReplicaStranded} { t.Run(refusal.Error(), func(t *testing.T) { mgr := &fakeManager{setVolumes: func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) { return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "old"}}, States: []types.VolumeState{{Name: "old", State: "pending", Error: cause}}, Error: "volume change failed; retry the desired target"}, fmt.Errorf("%w: volume %q", refusal, "data") From bae4c669ff282cc940e2cefa6f506911796add52 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 08:50:50 +0000 Subject: [PATCH 63/72] pool: reproduce stranded expiry and archive intent without a registry --- sandboxd/pool/replica_cleanup_test.go | 107 ++++++++++++++++++++------ 1 file changed, 82 insertions(+), 25 deletions(-) diff --git a/sandboxd/pool/replica_cleanup_test.go b/sandboxd/pool/replica_cleanup_test.go index cbeef1b2..6caf5995 100644 --- a/sandboxd/pool/replica_cleanup_test.go +++ b/sandboxd/pool/replica_cleanup_test.go @@ -3,42 +3,99 @@ package pool import ( "errors" "testing" + "time" "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestFailoverDisabledKeepsReplicaAndRefusesLiveWriter(t *testing.T) { - for _, pending := range []string{"published", "pending"} { - t.Run(pending, func(t *testing.T) { - m, c := newReplicaVolumeManager(t) - sb := mustClaim(t, m, testKey) - if pending == "pending" { - m.archives = &hookRegistry{Registry: c.reg, failPut: true} - if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { - t.Fatal(err) +// failoverState leaves sb with replica or archive state from a failover node and returns its checkpoints. +type failoverState func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) []string + +var strandedStates = map[string]failoverState{ + "published replica": func(t *testing.T, m *Manager, _ *failoverCluster, sb *types.Sandbox) []string { + replicateNow(t, m, sb) + return []string{sb.ReplicaCk} + }, + "pending replica": func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) []string { + m.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + return []string{sb.ReplicaPending} + }, + "archive intent": func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) []string { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + m.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := m.archive(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + return []string{sb.ArchivePending} + }, +} + +func restartStranded(t *testing.T, setup failoverState) (*Manager, *types.Sandbox, []string) { + t.Helper() + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + checkpoints := setup(t, m, c, sb) + if checkpoints[0] == "" { + t.Fatal("fixture left no failover checkpoint") + } + restarted := newManagerWithoutRegistry(t, m, c) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + restored, ok := restarted.byID(sb.ID) + if !ok { + t.Fatal("startup without a registry lost the claim") + } + if restored.ReplicaCk != sb.ReplicaCk || restored.ReplicaPending != sb.ReplicaPending || restored.ArchivePending != sb.ArchivePending { + t.Fatalf("startup without a registry changed failover fields: replica=%q pending=%q archive=%q", restored.ReplicaCk, restored.ReplicaPending, restored.ArchivePending) + } + return restarted, restored, checkpoints +} + +func TestFailoverDisabledKeepsStateAndRefusesLiveWriter(t *testing.T) { + for name, setup := range strandedStates { + t.Run(name, func(t *testing.T) { + m, sb, checkpoints := restartStranded(t, setup) + for _, ck := range checkpoints { + if !ckExists(t, m, ck) || m.deleteMarked(ck) { + t.Fatalf("startup without a registry queued or deleted checkpoint %s", ck) } - } else { - replicateNow(t, m, sb) - } - replica, intent := sb.ReplicaCk, sb.ReplicaPending - restarted := newManagerWithoutRegistry(t, m, c) - if err := restarted.Reconcile(t.Context()); err != nil { - t.Fatal(err) - } - restored, ok := restarted.byID(sb.ID) - if !ok || restored.ReplicaCk != replica || restored.ReplicaPending != intent { - t.Fatalf("startup without a registry changed replica fields: ok=%t replica=%q pending=%q", ok, restored.ReplicaCk, restored.ReplicaPending) } - for _, ck := range []string{replica, intent} { - if ck != "" && (!ckExists(t, restarted, ck) || restarted.deleteMarked(ck)) { - t.Fatalf("startup without a registry queued or deleted checkpoint %s", ck) + if sb.HibernateSnap != "" { + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) } } want := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - view, err := restarted.SetVolumes(t.Context(), sb.ID, "", want) + view, err := m.SetVolumes(t.Context(), sb.ID, "", want) if !errors.Is(err, ErrReplicaStranded) || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { - t.Fatalf("live writer on a stranded replica: err=%v volumes=%v want=%v", err, view.Volumes, view.VolumesWant) + t.Fatalf("live writer on stranded failover state: err=%v volumes=%v want=%v", err, view.Volumes, view.VolumesWant) + } + }) + } +} + +func TestFailoverDisabledExpiryDestroysClaimAndKeepsCheckpoint(t *testing.T) { + for _, name := range []string{"published replica", "pending replica"} { + t.Run(name, func(t *testing.T) { + m, sb, checkpoints := restartStranded(t, strandedStates[name]) + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Minute) + m.mu.Unlock() + for range 2 { + m.reapOnce(t.Context()) + } + if _, ok := m.byID(sb.ID); ok { + t.Fatal("expired claim with failover state outlived the reaper") + } + if !ckExists(t, m, checkpoints[0]) { + t.Fatal("expiry without a registry deleted a checkpoint another node may own") } }) } From d3e2de194080c7b2e1d7fe3e068f6deecf29ce70 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 08:52:25 +0000 Subject: [PATCH 64/72] pool: destroy expired claims whose replica cannot retire Without a registry, expiry routed a claim with replica state through dropReplica, which cannot settle a pending intent and deletes a published replica checkpoint that another node may have adopted. An expired ReplicaPending claim therefore never left the reaper. Expiry now drops rows only when a registry exists, so such claims are destroyed and their checkpoints stay on disk. Live writers also treat an unfinished archive intent as stranded state, since nothing retires it without a registry. --- docs/deploy.md | 15 ++++++++++----- docs/sandboxd-api.md | 12 ++++++------ docs/volume-controller.md | 2 +- sandboxd/pool/failover.go | 2 +- sandboxd/pool/pool.go | 2 +- sandboxd/pool/volume_mutation.go | 4 ++-- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 64352ecf..504f2a01 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -170,11 +170,16 @@ fragment the warm pools): ### Disabling archive failover Set `replicate_seconds: 0` while `archive_failover` is still enabled, then -restart, so the replica scan retires existing replicas. A claim that still has -a replica when `archive_failover` is disabled keeps it. sandboxd does not -delete that checkpoint or touch the remote registry row, and live writable -volume additions on the claim return 409 until it is released or failover is -re-enabled. +restart, so the replica scan retires existing replicas. Disabling the registry +does not consume remote rows or fence other nodes, so keep peers that still run +with `archive_failover` from adopting this node's claims until their rows are +drained. + +A claim that still has a replica or an unfinished archive transition when +`archive_failover` is disabled keeps that state. Live writable volume additions +on it return 409 until failover is re-enabled or the claim is released. When +such a claim expires it is destroyed, and its replica checkpoint stays on disk +until failover is re-enabled. ### Reading sleeping claims without waking them diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 8fa58315..a35e44f4 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -544,12 +544,12 @@ A non-replicating pool can still have an existing or pending replica after a restart. Live RW additions return 503 with `Retry-After` until that claim's replicas and recovery checkpoint references have retired. The background scan retires existing replicas; keep the controller's reservation and access gate -closed, then retry the request. Without `archive_failover` nothing retires a -leftover replica, so live RW additions on that claim return 409 instead; see -[disabling archive failover](deploy.md#disabling-archive-failover). A failed -claim write retains the affected checkpoint reference until a successful -snapshot acknowledges its removal. Unrelated claims' writes and checkpoint -references do not refuse this attachment. +closed, then retry the request. A failed claim write retains the affected +checkpoint reference until a successful snapshot acknowledges its removal. +Unrelated claims' writes and checkpoint references do not refuse this +attachment. Without `archive_failover` nothing retires a leftover replica or +archive intent, so live RW additions on that claim return 409 instead; see +[disabling archive failover](deploy.md#disabling-archive-failover). A retry of an admitted desired volume completes that operation without creating a new admission. Its volume hold continues to prevent replica publication. diff --git a/docs/volume-controller.md b/docs/volume-controller.md index ee9e3f9f..41f35f39 100644 --- a/docs/volume-controller.md +++ b/docs/volume-controller.md @@ -22,7 +22,7 @@ replica or recovery checkpoint reference, which returns 503 with `Retry-After`. Keep the reservation and access gate closed and retry attachment after that claim's retirement has been acknowledged. A claim that keeps a replica after [archive failover is disabled](deploy.md#disabling-archive-failover) returns 409 -and must be released. +until failover is re-enabled or the claim is released. A fixed delay does not acknowledge retirement. See [the live-volume retirement contract](sandboxd-api.md#partial-failure-and-recovery). diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 5c9bb915..7246e222 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -657,5 +657,5 @@ func (m *Manager) archiveRowed(sb *types.Sandbox) bool { } func (m *Manager) expiryDropsRow(sb *types.Sandbox) bool { - return m.archiveRowed(sb) || sb.HasReplica() && !m.archivesAtDeadline(sb) + return m.archiveRowed(sb) || m.archives != nil && sb.HasReplica() && !m.archivesAtDeadline(sb) } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 9e4ddb0d..2798d610 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -105,7 +105,7 @@ var ( ErrReplicaPresent = errors.New("live writable attachment requires replica and recovery checkpoint retirement") ErrReplicatingPool = errors.New("live writable volumes require a non-replicating pool") - ErrReplicaStranded = errors.New("claim keeps a replica from a failover configuration; release it or re-enable archive_failover") + ErrReplicaStranded = errors.New("claim keeps replica or archive state that only archive_failover can retire; re-enable it or release the claim") // Replaying a journal takes a writable mount, so readers stay out. ErrVolumeNeedsRecovery = errors.New("volume needs recovery by a writable claim") diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 1c59db9a..20c1eb1a 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -331,8 +331,8 @@ func (m *Manager) liveWriterError(sb *types.Sandbox) error { if m.view.Load().poolReplicate[sb.PolicyKey()] > 0 { return ErrReplicatingPool } - if sb.HasReplica() && m.archives == nil { - // only the replica scan retires a replica, and it needs the registry + if m.archives == nil && (sb.HasReplica() || sb.ArchivePending != "") { + // only registry-backed scans retire replica and archive intents return ErrReplicaStranded } if sb.HasReplica() || len(m.store.checkpointRefs(sb.ID)) > 0 { From 5db424ffb667f340a351dae648aadb75d4bfa08c Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 08:57:45 +0000 Subject: [PATCH 65/72] pool: reproduce replica deletion when expiry archives without a registry --- sandboxd/pool/replica_cleanup_test.go | 93 +++++++++++++++------------ 1 file changed, 52 insertions(+), 41 deletions(-) diff --git a/sandboxd/pool/replica_cleanup_test.go b/sandboxd/pool/replica_cleanup_test.go index 6caf5995..fbabe0ae 100644 --- a/sandboxd/pool/replica_cleanup_test.go +++ b/sandboxd/pool/replica_cleanup_test.go @@ -9,22 +9,25 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// failoverState leaves sb with replica or archive state from a failover node and returns its checkpoints. -type failoverState func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) []string +type strandedState struct { + name string + setup func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string + hibernate bool +} -var strandedStates = map[string]failoverState{ - "published replica": func(t *testing.T, m *Manager, _ *failoverCluster, sb *types.Sandbox) []string { +var strandedStates = []strandedState{ + {name: "published replica", setup: func(t *testing.T, m *Manager, _ *failoverCluster, sb *types.Sandbox) string { replicateNow(t, m, sb) - return []string{sb.ReplicaCk} - }, - "pending replica": func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) []string { + return sb.ReplicaCk + }}, + {name: "pending replica", setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { m.archives = &hookRegistry{Registry: c.reg, failPut: true} if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { t.Fatal(err) } - return []string{sb.ReplicaPending} - }, - "archive intent": func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) []string { + return sb.ReplicaPending + }}, + {name: "archive intent", hibernate: true, setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatal(err) } @@ -32,16 +35,16 @@ var strandedStates = map[string]failoverState{ if err := m.archive(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { t.Fatal(err) } - return []string{sb.ArchivePending} - }, + return sb.ArchivePending + }}, } -func restartStranded(t *testing.T, setup failoverState) (*Manager, *types.Sandbox, []string) { +func restartStranded(t *testing.T, state strandedState) (*Manager, *types.Sandbox, string) { t.Helper() m, c := newReplicaVolumeManager(t) sb := mustClaim(t, m, testKey) - checkpoints := setup(t, m, c, sb) - if checkpoints[0] == "" { + ck := state.setup(t, m, c, sb) + if ck == "" { t.Fatal("fixture left no failover checkpoint") } restarted := newManagerWithoutRegistry(t, m, c) @@ -55,19 +58,17 @@ func restartStranded(t *testing.T, setup failoverState) (*Manager, *types.Sandbo if restored.ReplicaCk != sb.ReplicaCk || restored.ReplicaPending != sb.ReplicaPending || restored.ArchivePending != sb.ArchivePending { t.Fatalf("startup without a registry changed failover fields: replica=%q pending=%q archive=%q", restored.ReplicaCk, restored.ReplicaPending, restored.ArchivePending) } - return restarted, restored, checkpoints + if !ckExists(t, restarted, ck) || restarted.deleteMarked(ck) { + t.Fatalf("startup without a registry queued or deleted checkpoint %s", ck) + } + return restarted, restored, ck } func TestFailoverDisabledKeepsStateAndRefusesLiveWriter(t *testing.T) { - for name, setup := range strandedStates { - t.Run(name, func(t *testing.T) { - m, sb, checkpoints := restartStranded(t, setup) - for _, ck := range checkpoints { - if !ckExists(t, m, ck) || m.deleteMarked(ck) { - t.Fatalf("startup without a registry queued or deleted checkpoint %s", ck) - } - } - if sb.HibernateSnap != "" { + for _, state := range strandedStates { + t.Run(state.name, func(t *testing.T) { + m, sb, _ := restartStranded(t, state) + if state.hibernate { if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatal(err) } @@ -81,23 +82,33 @@ func TestFailoverDisabledKeepsStateAndRefusesLiveWriter(t *testing.T) { } } -func TestFailoverDisabledExpiryDestroysClaimAndKeepsCheckpoint(t *testing.T) { - for _, name := range []string{"published replica", "pending replica"} { - t.Run(name, func(t *testing.T) { - m, sb, checkpoints := restartStranded(t, strandedStates[name]) - m.mu.Lock() - sb.Deadline = time.Now().Add(-time.Minute) - m.mu.Unlock() - for range 2 { +func TestFailoverDisabledExpiryKeepsReplicaCheckpoint(t *testing.T) { + for _, state := range strandedStates[:2] { + for _, onExpire := range []types.ExpireAction{types.ExpireDestroy, types.ExpireArchive} { + t.Run(state.name+"/"+string(onExpire), func(t *testing.T) { + m, sb, ck := restartStranded(t, state) + m.mu.Lock() + sb.OnExpire, sb.Deadline = onExpire, time.Now().Add(-time.Minute) + m.mu.Unlock() m.reapOnce(t.Context()) - } - if _, ok := m.byID(sb.ID); ok { - t.Fatal("expired claim with failover state outlived the reaper") - } - if !ckExists(t, m, checkpoints[0]) { - t.Fatal("expiry without a registry deleted a checkpoint another node may own") - } - }) + var archived bool + for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(10 * time.Millisecond) { + m.mu.Lock() + _, busy := m.archiving[sb.ID] + archived = sb.ArchiveCk != "" + m.mu.Unlock() + if !busy || time.Now().After(deadline) { + break + } + } + if _, held := m.byID(sb.ID); held != archived || archived != (onExpire == types.ExpireArchive) { + t.Fatalf("expiry with on_expire=%s: held=%t archived=%t", onExpire, held, archived) + } + if !ckExists(t, m, ck) { + t.Fatal("expiry without a registry deleted a replica checkpoint another node may own") + } + }) + } } } From e54696b475c77e2c7c6a705856d5d862c04eb9b6 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 08:59:23 +0000 Subject: [PATCH 66/72] pool: keep replica checkpoints when archiving without a registry A claim that archived at its deadline on a node without a registry deleted its replica checkpoint, which a peer may still own through the registry row. Without a registry, archive now leaves that checkpoint on disk, matching release and expiry, and the docs say so instead of promising automatic cleanup. --- docs/deploy.md | 17 +++++++++-------- docs/volume-controller.md | 7 ++++--- sandboxd/pool/archive.go | 3 ++- sandboxd/pool/volume_mutation.go | 2 +- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 504f2a01..c8f2b017 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -170,16 +170,17 @@ fragment the warm pools): ### Disabling archive failover Set `replicate_seconds: 0` while `archive_failover` is still enabled, then -restart, so the replica scan retires existing replicas. Disabling the registry -does not consume remote rows or fence other nodes, so keep peers that still run -with `archive_failover` from adopting this node's claims until their rows are -drained. +restart, so the replica scan retires existing replicas. Disabling +`archive_failover` does not consume remote rows or fence other nodes. Drain +this node's rows from a node that still has the registry, or keep peers that +run with `archive_failover` from adopting its claims. A claim that still has a replica or an unfinished archive transition when -`archive_failover` is disabled keeps that state. Live writable volume additions -on it return 409 until failover is re-enabled or the claim is released. When -such a claim expires it is destroyed, and its replica checkpoint stays on disk -until failover is re-enabled. +`archive_failover` is disabled keeps those fields. Live writable volume +additions on it return 409 until failover is re-enabled or the claim is +released. Whether the claim is released, expires, or archives locally, sandboxd +leaves its replica checkpoint on disk. Remove that checkpoint only after no +node can adopt the claim's registry row. ### Reading sleeping claims without waking them diff --git a/docs/volume-controller.md b/docs/volume-controller.md index 41f35f39..43a47bf1 100644 --- a/docs/volume-controller.md +++ b/docs/volume-controller.md @@ -20,9 +20,10 @@ Live RW additions require `replicate_seconds: 0`; a replicating pool returns setting requires a daemon restart. A non-replicating pool may still hold an old replica or recovery checkpoint reference, which returns 503 with `Retry-After`. Keep the reservation and access gate closed and retry attachment after that -claim's retirement has been acknowledged. A claim that keeps a replica after -[archive failover is disabled](deploy.md#disabling-archive-failover) returns 409 -until failover is re-enabled or the claim is released. +claim's retirement has been acknowledged. A claim that keeps replica or +archive state after +[archive failover is disabled](deploy.md#disabling-archive-failover) returns +409 until failover is re-enabled or the claim is released. A fixed delay does not acknowledge retirement. See [the live-volume retirement contract](sandboxd-api.md#partial-failure-and-recovery). diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index e6380456..02b65ae3 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -173,7 +173,8 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { // the store ck is authoritative from here m.destroy(ctx, vmName) m.dropSnap(ctx, snap) - if replica != "" { + // without a registry a peer may still own this replica's row + if replica != "" && m.archives != nil { m.deleteOrphanArchiveCk(ctx, replica) } m.counters.archives.Add(1) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 20c1eb1a..410857aa 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -332,7 +332,7 @@ func (m *Manager) liveWriterError(sb *types.Sandbox) error { return ErrReplicatingPool } if m.archives == nil && (sb.HasReplica() || sb.ArchivePending != "") { - // only registry-backed scans retire replica and archive intents + // only registry-backed scans retire replicas and archive intents return ErrReplicaStranded } if sb.HasReplica() || len(m.store.checkpointRefs(sb.ID)) > 0 { From 1ce2cccd7bc405d1a63a8b9bded90cb102feea6e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 09:08:49 +0000 Subject: [PATCH 67/72] pool: reproduce stranded replica lost by a local archive --- sandboxd/pool/replica_cleanup_test.go | 104 +++++++++++++++++--------- 1 file changed, 70 insertions(+), 34 deletions(-) diff --git a/sandboxd/pool/replica_cleanup_test.go b/sandboxd/pool/replica_cleanup_test.go index fbabe0ae..fd663469 100644 --- a/sandboxd/pool/replica_cleanup_test.go +++ b/sandboxd/pool/replica_cleanup_test.go @@ -10,24 +10,27 @@ import ( ) type strandedState struct { - name string - setup func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string - hibernate bool + name string + setup func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string + // wake: the fixture leaves the claim hibernated. + wake bool + // lifecycle: also check the replica checkpoint through release, expiry, and local archive. + lifecycle bool } var strandedStates = []strandedState{ - {name: "published replica", setup: func(t *testing.T, m *Manager, _ *failoverCluster, sb *types.Sandbox) string { + {name: "published replica", lifecycle: true, setup: func(t *testing.T, m *Manager, _ *failoverCluster, sb *types.Sandbox) string { replicateNow(t, m, sb) return sb.ReplicaCk }}, - {name: "pending replica", setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { + {name: "pending replica", lifecycle: true, setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { m.archives = &hookRegistry{Registry: c.reg, failPut: true} if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { t.Fatal(err) } return sb.ReplicaPending }}, - {name: "archive intent", hibernate: true, setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { + {name: "archive intent", wake: true, setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatal(err) } @@ -64,48 +67,81 @@ func restartStranded(t *testing.T, state strandedState) (*Manager, *types.Sandbo return restarted, restored, ck } +func requireStrandedWriter(t *testing.T, m *Manager, sb *types.Sandbox) { + t.Helper() + want := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + view, err := m.SetVolumes(t.Context(), sb.ID, "", want) + if !errors.Is(err, ErrReplicaStranded) || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("live writer on stranded failover state: err=%v volumes=%v want=%v", err, view.Volumes, view.VolumesWant) + } +} + func TestFailoverDisabledKeepsStateAndRefusesLiveWriter(t *testing.T) { for _, state := range strandedStates { t.Run(state.name, func(t *testing.T) { m, sb, _ := restartStranded(t, state) - if state.hibernate { + if state.wake { if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatal(err) } } - want := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} - view, err := m.SetVolumes(t.Context(), sb.ID, "", want) - if !errors.Is(err, ErrReplicaStranded) || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { - t.Fatalf("live writer on stranded failover state: err=%v volumes=%v want=%v", err, view.Volumes, view.VolumesWant) - } + requireStrandedWriter(t, m, sb) }) } } -func TestFailoverDisabledExpiryKeepsReplicaCheckpoint(t *testing.T) { - for _, state := range strandedStates[:2] { - for _, onExpire := range []types.ExpireAction{types.ExpireDestroy, types.ExpireArchive} { - t.Run(state.name+"/"+string(onExpire), func(t *testing.T) { - m, sb, ck := restartStranded(t, state) +func TestFailoverDisabledLifecycleKeepsReplicaCheckpoint(t *testing.T) { + expire := func(m *Manager, sb *types.Sandbox, action types.ExpireAction) { + m.mu.Lock() + sb.OnExpire, sb.Deadline = action, time.Now().Add(-time.Minute) + m.mu.Unlock() + m.reapOnce(t.Context()) + } + exits := []struct { + name string + run func(t *testing.T, m *Manager, sb *types.Sandbox) + }{ + {"release", func(t *testing.T, m *Manager, sb *types.Sandbox) { + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + }}, + {"expire destroy", func(t *testing.T, m *Manager, sb *types.Sandbox) { + expire(m, sb, types.ExpireDestroy) + waitFor(t, func() bool { return m.Counters().Reaps == 1 }) + }}, + {"expire archive", func(t *testing.T, m *Manager, sb *types.Sandbox) { + expire(m, sb, types.ExpireArchive) + waitFor(t, func() bool { m.mu.Lock() - sb.OnExpire, sb.Deadline = onExpire, time.Now().Add(-time.Minute) - m.mu.Unlock() - m.reapOnce(t.Context()) - var archived bool - for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(10 * time.Millisecond) { - m.mu.Lock() - _, busy := m.archiving[sb.ID] - archived = sb.ArchiveCk != "" - m.mu.Unlock() - if !busy || time.Now().After(deadline) { - break - } - } - if _, held := m.byID(sb.ID); held != archived || archived != (onExpire == types.ExpireArchive) { - t.Fatalf("expiry with on_expire=%s: held=%t archived=%t", onExpire, held, archived) - } + defer m.mu.Unlock() + _, busy := m.archiving[sb.ID] + return sb.ArchiveCk != "" && !busy + }) + }}, + {"archive and wake", func(t *testing.T, m *Manager, sb *types.Sandbox) { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if err := m.archive(t.Context(), sb); err != nil { + t.Fatal(err) + } + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + requireStrandedWriter(t, m, sb) + }}, + } + for _, state := range strandedStates { + if !state.lifecycle { + continue + } + for _, exit := range exits { + t.Run(state.name+"/"+exit.name, func(t *testing.T) { + m, sb, ck := restartStranded(t, state) + exit.run(t, m, sb) if !ckExists(t, m, ck) { - t.Fatal("expiry without a registry deleted a replica checkpoint another node may own") + t.Fatalf("%s without a registry deleted a replica checkpoint another node may own", exit.name) } }) } From 7d29b6d46ec650a62e1ebc3d487e0cb52a77ad7d Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 09:10:25 +0000 Subject: [PATCH 68/72] pool: keep a stranded replica through a local archive Without a registry, archiving a claim cleared its replica field, so after a wake live writers were admitted while the replica's registry row still let a peer adopt the pre-attach image. The claim now keeps that replica through a local archive, so the 409 still applies after wake and the replica scan can retire it once failover is re-enabled. --- docs/deploy.md | 11 ++++++----- sandboxd/pool/archive.go | 7 +++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index c8f2b017..b90a79c9 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -176,11 +176,12 @@ this node's rows from a node that still has the registry, or keep peers that run with `archive_failover` from adopting its claims. A claim that still has a replica or an unfinished archive transition when -`archive_failover` is disabled keeps those fields. Live writable volume -additions on it return 409 until failover is re-enabled or the claim is -released. Whether the claim is released, expires, or archives locally, sandboxd -leaves its replica checkpoint on disk. Remove that checkpoint only after no -node can adopt the claim's registry row. +`archive_failover` is disabled keeps its replica, including through a local +archive. Live writable volume additions on it return 409 until failover is +re-enabled or the claim is released. Whether the claim is released, expires, or +archives locally, sandboxd leaves the replica checkpoint in the checkpoint +store. After failover is re-enabled, the replica scan retires a remaining +replica the next time the claim runs. ### Reading sleeping claims without waking them diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 02b65ae3..db33566f 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -145,7 +145,10 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { } } snap, vmName, sock, prevDeadline, prevPending, replica := sb.HibernateSnap, sb.VMName, sb.VsockSocket, sb.Deadline, sb.ArchivePending, sb.ReplicaCk - sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk = ck.ID, epoch, "", "" + sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending = ck.ID, epoch, "" + if m.archives != nil { + sb.ReplicaCk = "" + } sb.HibernateSnap, sb.VsockSocket, sb.VMName = "", "", "" sb.Deadline = m.retentionDeadline(sb.Key) js := m.store.set(sb) @@ -173,7 +176,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { // the store ck is authoritative from here m.destroy(ctx, vmName) m.dropSnap(ctx, snap) - // without a registry a peer may still own this replica's row + // without a registry a peer may still own this replica's row, so the claim keeps it if replica != "" && m.archives != nil { m.deleteOrphanArchiveCk(ctx, replica) } From 7ce47b15d8ea3fb014f98d82e22dcd85f665a86e Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 09:21:21 +0000 Subject: [PATCH 69/72] pool: leave local archive behaviour unchanged without a registry Keeping replica fields and checkpoints through a registry-less archive closed one admission gap but broke fencing and row cleanup once archive_failover was re-enabled, and left archive intents uncovered. Running without a registry while claims still carry replica state is an operator precondition violation that sandboxd cannot reason about, so archive keeps its upstream behaviour and the docs state the precondition and what each exit does. --- docs/deploy.md | 14 ++++----- sandboxd/pool/archive.go | 8 ++--- sandboxd/pool/replica_cleanup_test.go | 42 ++++++--------------------- 3 files changed, 18 insertions(+), 46 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index b90a79c9..b1b57d25 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -175,13 +175,13 @@ restart, so the replica scan retires existing replicas. Disabling this node's rows from a node that still has the registry, or keep peers that run with `archive_failover` from adopting its claims. -A claim that still has a replica or an unfinished archive transition when -`archive_failover` is disabled keeps its replica, including through a local -archive. Live writable volume additions on it return 409 until failover is -re-enabled or the claim is released. Whether the claim is released, expires, or -archives locally, sandboxd leaves the replica checkpoint in the checkpoint -store. After failover is re-enabled, the replica scan retires a remaining -replica the next time the claim runs. +Disable `archive_failover` only after those replicas have retired. sandboxd +does not support running without a registry while claims still carry replica +or archive state. If it does, live writable volume additions on such a claim +return 409 until failover is re-enabled or the claim is released, and release +or expiry leave the replica checkpoint in the checkpoint store. A local archive +of such a claim deletes the replica checkpoint and clears the replica, as in +earlier releases, so the 409 no longer applies after it wakes. ### Reading sleeping claims without waking them diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index db33566f..e6380456 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -145,10 +145,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { } } snap, vmName, sock, prevDeadline, prevPending, replica := sb.HibernateSnap, sb.VMName, sb.VsockSocket, sb.Deadline, sb.ArchivePending, sb.ReplicaCk - sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending = ck.ID, epoch, "" - if m.archives != nil { - sb.ReplicaCk = "" - } + sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk = ck.ID, epoch, "", "" sb.HibernateSnap, sb.VsockSocket, sb.VMName = "", "", "" sb.Deadline = m.retentionDeadline(sb.Key) js := m.store.set(sb) @@ -176,8 +173,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { // the store ck is authoritative from here m.destroy(ctx, vmName) m.dropSnap(ctx, snap) - // without a registry a peer may still own this replica's row, so the claim keeps it - if replica != "" && m.archives != nil { + if replica != "" { m.deleteOrphanArchiveCk(ctx, replica) } m.counters.archives.Add(1) diff --git a/sandboxd/pool/replica_cleanup_test.go b/sandboxd/pool/replica_cleanup_test.go index fd663469..f512e012 100644 --- a/sandboxd/pool/replica_cleanup_test.go +++ b/sandboxd/pool/replica_cleanup_test.go @@ -14,7 +14,7 @@ type strandedState struct { setup func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string // wake: the fixture leaves the claim hibernated. wake bool - // lifecycle: also check the replica checkpoint through release, expiry, and local archive. + // lifecycle: also check that release and expiry keep the replica checkpoint. lifecycle bool } @@ -90,13 +90,7 @@ func TestFailoverDisabledKeepsStateAndRefusesLiveWriter(t *testing.T) { } } -func TestFailoverDisabledLifecycleKeepsReplicaCheckpoint(t *testing.T) { - expire := func(m *Manager, sb *types.Sandbox, action types.ExpireAction) { - m.mu.Lock() - sb.OnExpire, sb.Deadline = action, time.Now().Add(-time.Minute) - m.mu.Unlock() - m.reapOnce(t.Context()) - } +func TestFailoverDisabledExitKeepsReplicaCheckpoint(t *testing.T) { exits := []struct { name string run func(t *testing.T, m *Manager, sb *types.Sandbox) @@ -106,31 +100,13 @@ func TestFailoverDisabledLifecycleKeepsReplicaCheckpoint(t *testing.T) { t.Fatal(err) } }}, - {"expire destroy", func(t *testing.T, m *Manager, sb *types.Sandbox) { - expire(m, sb, types.ExpireDestroy) + {"expire", func(t *testing.T, m *Manager, sb *types.Sandbox) { + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Minute) + m.mu.Unlock() + m.reapOnce(t.Context()) waitFor(t, func() bool { return m.Counters().Reaps == 1 }) }}, - {"expire archive", func(t *testing.T, m *Manager, sb *types.Sandbox) { - expire(m, sb, types.ExpireArchive) - waitFor(t, func() bool { - m.mu.Lock() - defer m.mu.Unlock() - _, busy := m.archiving[sb.ID] - return sb.ArchiveCk != "" && !busy - }) - }}, - {"archive and wake", func(t *testing.T, m *Manager, sb *types.Sandbox) { - if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { - t.Fatal(err) - } - if err := m.archive(t.Context(), sb); err != nil { - t.Fatal(err) - } - if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { - t.Fatal(err) - } - requireStrandedWriter(t, m, sb) - }}, } for _, state := range strandedStates { if !state.lifecycle { @@ -140,8 +116,8 @@ func TestFailoverDisabledLifecycleKeepsReplicaCheckpoint(t *testing.T) { t.Run(state.name+"/"+exit.name, func(t *testing.T) { m, sb, ck := restartStranded(t, state) exit.run(t, m, sb) - if !ckExists(t, m, ck) { - t.Fatalf("%s without a registry deleted a replica checkpoint another node may own", exit.name) + if _, held := m.byID(sb.ID); held || !ckExists(t, m, ck) { + t.Fatalf("%s without a registry: held=%t checkpoint kept=%t", exit.name, held, ckExists(t, m, ck)) } }) } From 46e5bf36c3d1b442d1dcf6ba2623fb13e436b1ef Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 09:24:05 +0000 Subject: [PATCH 70/72] docs: scope the local archive note to published replicas --- docs/deploy.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index b1b57d25..902b1c06 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -180,8 +180,9 @@ does not support running without a registry while claims still carry replica or archive state. If it does, live writable volume additions on such a claim return 409 until failover is re-enabled or the claim is released, and release or expiry leave the replica checkpoint in the checkpoint store. A local archive -of such a claim deletes the replica checkpoint and clears the replica, as in -earlier releases, so the 409 no longer applies after it wakes. +of a claim with a published replica deletes that checkpoint and clears the +replica, as in earlier releases, so the 409 no longer applies after it wakes. +A pending replica survives the archive and keeps the 409. ### Reading sleeping claims without waking them From ef520f7c3f190b9dba35dff4683002aaab9c6f73 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 09:50:03 +0000 Subject: [PATCH 71/72] pool: clarify no-registry replica recovery guidance --- docs/deploy.md | 27 +++++++++++++++++++-------- docs/sandboxd-api.md | 7 ++++--- docs/volume-controller.md | 8 ++++---- sandboxd/pool/pool.go | 2 +- sandboxd/pool/replica_cleanup_test.go | 19 +++++-------------- sandboxd/pool/volume_mutation.go | 1 - 6 files changed, 33 insertions(+), 31 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 902b1c06..f57a63f6 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -169,20 +169,31 @@ fragment the warm pools): ### Disabling archive failover +Before changing `archive_failover`, verify local claim ownership and the +corresponding remote rows from a node with registry access. Set `replicate_seconds: 0` while `archive_failover` is still enabled, then restart, so the replica scan retires existing replicas. Disabling `archive_failover` does not consume remote rows or fence other nodes. Drain this node's rows from a node that still has the registry, or keep peers that run with `archive_failover` from adopting its claims. -Disable `archive_failover` only after those replicas have retired. sandboxd -does not support running without a registry while claims still carry replica -or archive state. If it does, live writable volume additions on such a claim -return 409 until failover is re-enabled or the claim is released, and release -or expiry leave the replica checkpoint in the checkpoint store. A local archive -of a claim with a published replica deletes that checkpoint and clears the -replica, as in earlier releases, so the 409 no longer applies after it wakes. -A pending replica survives the archive and keeps the 409. +Disable `archive_failover` only after those replicas have retired. If claims +still carry `ReplicaCk`, `ReplicaPending`, or `ArchivePending`, startup without +a registry retains those fields and refuses new live writable volume additions +with 409 while any of them remain. Release and default destroy-on-expiry leave +replica checkpoints in the checkpoint store. A successful local archive, +including archive-on-expiry, clears `ReplicaCk` and `ArchivePending` and queues +the published replica checkpoint for deletion. It retains `ReplicaPending`, +which continues to refuse live writable additions after wake. Once the current +fields clear, recovery checkpoint references can still require temporary 503 +retirement before admission. + +Re-enabling `archive_failover` does not guarantee recovery of a retained claim. +Before re-enabling it, verify that a published replica's row still exists, +belongs to this node, and can be retired. If the row is absent, was drained, or +has moved to another owner, choose an explicit recovery procedure before the +transition. Startup fencing can drop the local claim, and reconciliation can +remove its VM, losing changes since the last checkpoint. ### Reading sleeping claims without waking them diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index a35e44f4..1b9cfeb7 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -547,9 +547,10 @@ retires existing replicas; keep the controller's reservation and access gate closed, then retry the request. A failed claim write retains the affected checkpoint reference until a successful snapshot acknowledges its removal. Unrelated claims' writes and checkpoint references do not refuse this -attachment. Without `archive_failover` nothing retires a leftover replica or -archive intent, so live RW additions on that claim return 409 instead; see -[disabling archive failover](deploy.md#disabling-archive-failover). +attachment. Without `archive_failover`, live RW additions return 409 while the +claim retains replica or archive intent fields. See +[disabling archive failover](deploy.md#disabling-archive-failover) for transition +checks, local archive exceptions, and recovery risks. A retry of an admitted desired volume completes that operation without creating a new admission. Its volume hold continues to prevent replica publication. diff --git a/docs/volume-controller.md b/docs/volume-controller.md index 43a47bf1..601ae662 100644 --- a/docs/volume-controller.md +++ b/docs/volume-controller.md @@ -20,10 +20,10 @@ Live RW additions require `replicate_seconds: 0`; a replicating pool returns setting requires a daemon restart. A non-replicating pool may still hold an old replica or recovery checkpoint reference, which returns 503 with `Retry-After`. Keep the reservation and access gate closed and retry attachment after that -claim's retirement has been acknowledged. A claim that keeps replica or -archive state after -[archive failover is disabled](deploy.md#disabling-archive-failover) returns -409 until failover is re-enabled or the claim is released. +claim's retirement has been acknowledged. With archive failover disabled, +retained replica or archive intent fields refuse live RW additions with 409. +Follow [disabling archive failover](deploy.md#disabling-archive-failover) for +transition checks, local archive exceptions, and recovery risks. A fixed delay does not acknowledge retirement. See [the live-volume retirement contract](sandboxd-api.md#partial-failure-and-recovery). diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 2798d610..883451f0 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -105,7 +105,7 @@ var ( ErrReplicaPresent = errors.New("live writable attachment requires replica and recovery checkpoint retirement") ErrReplicatingPool = errors.New("live writable volumes require a non-replicating pool") - ErrReplicaStranded = errors.New("claim keeps replica or archive state that only archive_failover can retire; re-enable it or release the claim") + ErrReplicaStranded = errors.New("live writable attachment is blocked by retained replica or archive state with archive_failover disabled") // Replaying a journal takes a writable mount, so readers stay out. ErrVolumeNeedsRecovery = errors.New("volume needs recovery by a writable claim") diff --git a/sandboxd/pool/replica_cleanup_test.go b/sandboxd/pool/replica_cleanup_test.go index f512e012..9b81ead2 100644 --- a/sandboxd/pool/replica_cleanup_test.go +++ b/sandboxd/pool/replica_cleanup_test.go @@ -12,25 +12,21 @@ import ( type strandedState struct { name string setup func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string - // wake: the fixture leaves the claim hibernated. - wake bool - // lifecycle: also check that release and expiry keep the replica checkpoint. - lifecycle bool } var strandedStates = []strandedState{ - {name: "published replica", lifecycle: true, setup: func(t *testing.T, m *Manager, _ *failoverCluster, sb *types.Sandbox) string { + {name: "published replica", setup: func(t *testing.T, m *Manager, _ *failoverCluster, sb *types.Sandbox) string { replicateNow(t, m, sb) return sb.ReplicaCk }}, - {name: "pending replica", lifecycle: true, setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { + {name: "pending replica", setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { m.archives = &hookRegistry{Registry: c.reg, failPut: true} if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { t.Fatal(err) } return sb.ReplicaPending }}, - {name: "archive intent", wake: true, setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { + {name: "archive intent", setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatal(err) } @@ -80,10 +76,8 @@ func TestFailoverDisabledKeepsStateAndRefusesLiveWriter(t *testing.T) { for _, state := range strandedStates { t.Run(state.name, func(t *testing.T) { m, sb, _ := restartStranded(t, state) - if state.wake { - if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { - t.Fatal(err) - } + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) } requireStrandedWriter(t, m, sb) }) @@ -109,9 +103,6 @@ func TestFailoverDisabledExitKeepsReplicaCheckpoint(t *testing.T) { }}, } for _, state := range strandedStates { - if !state.lifecycle { - continue - } for _, exit := range exits { t.Run(state.name+"/"+exit.name, func(t *testing.T) { m, sb, ck := restartStranded(t, state) diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 410857aa..f973573b 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -332,7 +332,6 @@ func (m *Manager) liveWriterError(sb *types.Sandbox) error { return ErrReplicatingPool } if m.archives == nil && (sb.HasReplica() || sb.ArchivePending != "") { - // only registry-backed scans retire replicas and archive intents return ErrReplicaStranded } if sb.HasReplica() || len(m.store.checkpointRefs(sb.ID)) > 0 { From 53852091034546fe1c3f9e813bc189980b660327 Mon Sep 17 00:00:00 2001 From: minatoaquaMK2 Date: Sun, 25 Oct 2026 18:14:37 +0800 Subject: [PATCH 72/72] docs: keep volume backings outside cleanup directories --- docs/sandboxd-api.md | 6 ++++++ docs/volume-controller.md | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 1b9cfeb7..8ae30d2b 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -307,6 +307,12 @@ JSON return 400. A missing backing retains its registration but cannot be attached until available. Registration success does not establish that Cocoon supports the backing type. Operators must keep the registered path and target stable. + Backing storage must be separate from daemon cleanup directories, including + `/readers`, `/goldens`, and `/fork-*`. + Backing paths must not resolve into cleanup directories. + Cleanup directories must not resolve into backing storage. + Startup and staging cleanup do not consult volume registrations and can + delete backing files placed in those locations. - `directio` is `on`, `off`, or `auto`, and defaults to `off`. - `writable` defaults to `false` and controls writable claim admission. - `tenants` grants the named tenants self-service access. Omitted or empty diff --git a/docs/volume-controller.md b/docs/volume-controller.md index 601ae662..84291f72 100644 --- a/docs/volume-controller.md +++ b/docs/volume-controller.md @@ -36,8 +36,14 @@ Use the current Python SDK with [per-entry attach-only support](sdk-python.md#changing-volumes), Cocoon v0.7.2 or newer, and Cloud Hypervisor. -Prepare and format the backing outside sandboxd. Register it on the claim's -node with the configured root token: +Prepare and format the backing outside sandboxd. Store it in a dedicated +persistent directory, such as `/srv/volumes`. Keep backing storage separate +from daemon cleanup directories, including `/readers`, +`/goldens`, and `/fork-*`, and prevent aliases between them. +Startup and staging cleanup can delete a backing in those locations even +while it remains registered. + +Register the backing on the claim's node with the configured root token: ```http PUT /v1/volumes/home