diff --git a/.github/workflows/sandboxd.yml b/.github/workflows/sandboxd.yml index fdbba839..16321074 100644 --- a/.github/workflows/sandboxd.yml +++ b/.github/workflows/sandboxd.yml @@ -67,6 +67,29 @@ jobs: - name: Test run: make go-test + - name: Test pool as root + working-directory: sandboxd + shell: bash + run: | + GOWORK=off go test -race -c -o "$RUNNER_TEMP/catalog.test" ./pool + sudo setpriv --bounding-set=-dac_override,-dac_read_search "$RUNNER_TEMP/catalog.test" -test.count=1 -test.v -test.skip '^Test(PromoteFailsClosedOnMetaError|NodesOfACellShareThePoolSet|AMetaStoreNodeWithoutACellKeepsItsOwnPools|TwoNodesOnOneMetaStoreShareTheTenantSet)$' | tee "$RUNNER_TEMP/catalog-test.log" + grep -q '^--- PASS: TestRuntimeCatalogPersistsNormalizedSemantics ' "$RUNNER_TEMP/catalog-test.log" + if grep -- '--- SKIP:' "$RUNNER_TEMP/catalog-test.log"; then + exit 1 + fi + + - name: Test e2e as root + working-directory: e2e + shell: bash + run: | + GOWORK=off go test -race -c -o "$RUNNER_TEMP/volume-e2e.test" . + sudo setpriv --bounding-set=-dac_override,-dac_read_search "$RUNNER_TEMP/volume-e2e.test" -test.count=1 -test.v -test.skip '^TestCaddyTLSCluster$' | tee "$RUNNER_TEMP/volume-e2e-test.log" + grep -q '^--- PASS: TestRuntimeVolumeControllerAdmissionThroughHTTP ' "$RUNNER_TEMP/volume-e2e-test.log" + grep -q '^--- PASS: TestRuntimeVolumeExplicitACLAndStaticPublicThroughHTTP ' "$RUNNER_TEMP/volume-e2e-test.log" + if grep -- '--- SKIP:' "$RUNNER_TEMP/volume-e2e-test.log"; then + exit 1 + fi + - name: Test the PostgreSQL meta store working-directory: sandboxd shell: bash @@ -75,7 +98,9 @@ jobs: run: | GOWORK=off go test -race -count=1 -v ./tenants/... ./poolset/... | tee "$RUNNER_TEMP/pg-test.log" GOWORK=off go test -race -count=1 -v -run 'TestTwoNodesOnOneMetaStore|TestNodesOfACell|TestAMetaStoreNodeWithoutACell' ./pool/ | tee -a "$RUNNER_TEMP/pg-test.log" - ! grep -- '--- SKIP' "$RUNNER_TEMP/pg-test.log" + if grep -- '--- SKIP:' "$RUNNER_TEMP/pg-test.log"; then + exit 1 + fi - name: Set up Python uses: actions/setup-python@v7.0.0 diff --git a/README.md b/README.md index 04468e8d..68d48586 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ performance) — source in - `sandboxd/` — per-node control plane (Go): warm pools refilled from golden snapshot exports (online-retunable), claim/release/renew/hibernate/fork/ promote/checkpoint HTTP API, operator-catalog dataset volumes (read-only or - writable; read-only ones change on a running claim), signed preview URLs, + writable, mounted or attach-only; all support live changes), signed preview URLs, the HTTP-upgrade byte relays to silkd and to any guest port, read-only file access to hibernated or archived claims from a warm VM of their pool (opt-in per pool `reader`), claim env with diff --git a/docs/deploy.md b/docs/deploy.md index f8020f0e..f57a63f6 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -167,6 +167,34 @@ fragment the warm pools): | `xlarge` | 4 | 8G | | `2xlarge` | 8 | 16G | +### Disabling archive failover + +Before changing `archive_failover`, verify local claim ownership and the +corresponding remote rows from a node with registry access. +Set `replicate_seconds: 0` while `archive_failover` is still enabled, then +restart, so the replica scan retires existing replicas. Disabling +`archive_failover` does not consume remote rows or fence other nodes. Drain +this node's rows from a node that still has the registry, or keep peers that +run with `archive_failover` from adopting its claims. + +Disable `archive_failover` only after those replicas have retired. If claims +still carry `ReplicaCk`, `ReplicaPending`, or `ArchivePending`, startup without +a registry retains those fields and refuses new live writable volume additions +with 409 while any of them remain. Release and default destroy-on-expiry leave +replica checkpoints in the checkpoint store. A successful local archive, +including archive-on-expiry, clears `ReplicaCk` and `ArchivePending` and queues +the published replica checkpoint for deletion. It retains `ReplicaPending`, +which continues to refuse live writable additions after wake. Once the current +fields clear, recovery checkpoint references can still require temporary 503 +retirement before admission. + +Re-enabling `archive_failover` does not guarantee recovery of a retained claim. +Before re-enabling it, verify that a published replica's row still exists, +belongs to this node, and can be retired. If the row is absent, was drained, or +has moved to another owner, choose an explicit recovery procedure before the +transition. Startup fencing can drop the local claim, and reconciliation can +remove its VM, losing changes since the last checkpoint. + ### Reading sleeping claims without waking them A client that only lists or reads files (a UI showing an idle agent's @@ -247,7 +275,14 @@ the pool had `reader`, and its reads wake the claim as without the feature. ### Dataset volumes -Running claims can change mounted read-only catalog entries through the +Volume `tenants` entries in static configuration must match the tenant name +syntax used by the tenant API. Configurations containing invalid names that +previously only produced warnings now fail validation at startup. Remove or +correct those entries before upgrading. Valid names for tenants that have not +yet been created remain accepted. + +Running claims can add or remove mounted and attach-only catalog entries in +read-only or writable mode through the [GET, PUT, and PATCH volume API](sandboxd-api.md#get-v1sandboxesidvolumes) and both SDKs. The image must already exist on the owner node. Live changes require Cocoon v0.7.2 or newer for idempotent disk attach and detach. PUT and @@ -260,8 +295,8 @@ The claim journal stores confirmed applied `Volumes`, plus desired `VolumesWant` while it differs. Admission holds cover their union across daemon restart and teardown. Retry an interrupted operation with the same PUT target. Release cancels convergence -and transfers holds to teardown. Capture is refused while either set is -nonempty. +and transfers holds to teardown. Capture is refused while either set, an active +operation, or unacknowledged recovery references still hold volumes. A sandbox whose VMM exits while holding volumes is release-only. Release it and create a new sandbox. Daemon restart recovery assumes the original VMM remains @@ -272,8 +307,9 @@ interrupted diagnostic after restart and require a caller retry. exercises GET, repeated full-set PUT, PATCH add and remove, and clearing a new claim. -Each catalog path must name a disk image containing a mountable whole-device -filesystem. Keep catalog names bound to the same image while any claim holds +Mounted entries need a disk image containing a mountable whole-device +filesystem. Attach-only entries leave device use and filesystem consistency +to the caller. Keep catalog names bound to the same image while any claim holds them. Use versioned names such as `dataset-v1` and `dataset-v2`; sandboxd cannot detect a repointed catalog entry. A read-only entry's image must stay immutable: do not replace, @@ -304,9 +340,11 @@ A volume claim may consume an ordinary warm Cloud Hypervisor VM. sandboxd attaches after the warm pop or provision, waits for the device settle — a serial match under `/sys/block`, then the `/dev/` node itself, since the kernel publishes sysfs before devtmpfs creates it — up to 2 seconds, then -mounts the device before finalizing the claim: `mode: "ro"` (the default) -attaches and mounts read-only; `mode: "rw"` requires the catalog entry's -`writable: true` and attaches and mounts read-write. Setup failure destroys +mounts each managed device before finalizing the claim. `mode: "ro"` (the default) +attaches read-only; `mode: "rw"` requires the catalog entry's +`writable: true` and attaches read-write. Per-entry `attach_only: true`, or +claim-wide `volumes_attach_only: true`, skips the device wait and mount. The +caller must wait for enumeration before using that device. Setup failure destroys the VM without quiescing — the claim was never handed out, so no workload write happened — and a popped warm VM is refilled normally. @@ -314,8 +352,8 @@ A multi-volume claim brings every volume up concurrently: each volume's own marker→attach→mount order is preserved, but cocoon serializes the hypervisor attach per VM, so what actually overlaps across volumes is the CLI spawns, device settles, and guest mounts. A partial failure still fails the whole -claim and destroys the VM without quiescing, so any `rw` volume already -attached keeps its dirty marker — including one the caller never got a handle +claim and destroys the VM without quiescing, so any mounted `rw` volume already +attached keeps its dirty marker, including one the caller never got a handle to; recovery is the normal `rw` cycle (see [Writable dataset volumes](#writable-dataset-volumes)). @@ -331,19 +369,20 @@ boot. the answering node's current local availability and whether the entry is `writable`, without exposing host paths or node addresses. Applied names, effective mounts, and (for `rw` entries) mode are persisted with the claim. -Such a claim cannot hibernate, fork, checkpoint, or promote; the idle -hibernate sweep leaves it running. Release removes the VM but never deletes -the operator-owned backing image — an `rw` release quiesces (unmounts) first, +While volume holds or unacknowledged recovery references remain, a claim cannot +hibernate, fork, checkpoint, or promote. The idle hibernate sweep leaves it +running. Release removes the VM but never deletes +the operator-owned backing image. A mounted `rw` release quiesces (unmounts) first, below. With `directio=off`, readers share the host page cache; use `directio=on` when cache interference matters. A dataset mounted into an egress-lane sandbox can be uploaded wherever that tenant's egress policy permits, so treat the ACL and egress policy as one access -decision. Image replication, detach, and refcounting remain out of scope. +decision. Image replication and business ownership remain external to sandboxd. #### Writable dataset volumes -A catalog entry with `writable: true` may be claimed with `mode: "rw"` -(omitted, or `"ro"`, always mounts read-only, even against a writable entry). +A catalog entry with `writable: true` may be claimed with `mode: "rw"`. +Omitted mode, or `"ro"`, attaches read-only, even against a writable entry. `directio` still defaults to `off` for a writable entry — the durability nuance is that the host page cache sits between the guest's flush and the device, so `directio=on` is the knob when that gap matters, not just cache @@ -356,7 +395,7 @@ so every claim for that name, `ro` or `rw`, already funnels to that one node; running the same writable name from two nodes at once is dataset divergence that nothing in the protocol detects for you. -**The `.dirty` marker.** Before the first `rw` attach of an image, +**The `.dirty` marker.** Before a mounted `rw` attach of an image, sandboxd durably creates a `.dirty` sidecar beside it; a clean `rw` release removes it. It is a write-ahead record, not a lock: it survives a killed sandboxd or a dead VMM, so anything short of a clean unmount leaves it @@ -369,6 +408,13 @@ can serve `ro` again. A `ro` claim against a dirty image is refused rather than auto-healed: replaying the journal takes the write lock, which conflicts with concurrent readers. +Live removal clears a mounted writer's marker only with current-call evidence +of the expected device at the managed root mount, writable mount and +superblock options, normal unmount, no remaining device mounts, and confirmed +host detach. Marker cleanup precedes release of the old hold. A prior external +unmount or absent device provides no such evidence. Detach can succeed while +the marker remains and read-only use still requires recovery. + **Concurrency, for one image name:** - `rw` ∥ `rw` — refused (409). @@ -380,17 +426,13 @@ with concurrent readers. proceed; a crashed writer leaves it, and `ro` claims are refused until one `rw` claim replays and releases cleanly. -**Attach-only claims opt out of the marker, not out of admission.** A claim -sent with `volumes_attach_only` gets the device attached and nothing else, so -sandboxd neither writes nor clears `.dirty` for it — it cannot verify a -mount it did not perform. Everything in this section still applies to the -default, mounting claims, and the exclusion rules above apply to attach-only -claims exactly the same way, which is what keeps other tenants safe. The -operator-visible difference: an image whose attach-only writer released -without unmounting cleanly carries no marker, so the next `ro` claim is -admitted and fails at mount time (500) instead of being refused early (409). -Whoever hands out attach-only `rw` access owns that trade; see -[sandboxd-api](sandboxd-api.md#attach-only-volumes). +**Attach-only entries have no daemon-managed dirty marker.** This applies to +both initial claims and live additions. sandboxd neither writes nor clears +`.dirty` for attach-only `rw`, including a guest crash during writes. +Reader and writer exclusion still applies, but a later `ro` claim can be +admitted without a marker and may see an inconsistent filesystem. Mount failure +is not guaranteed. The controller owns quiescence, unmount, crash recovery, +and safe read-only publication. See the [attach-only contract](sandboxd-api.md#attach-only-volumes). ### Reclaiming S3 storage diff --git a/docs/live-volumes.md b/docs/live-volumes.md index 790e0d21..82e798cd 100644 --- a/docs/live-volumes.md +++ b/docs/live-volumes.md @@ -7,7 +7,22 @@ Register both read-only catalog entries on the owning node and grant the sandbox's tenant access to them. Put `v1` and `v2` in each image's `/version` file. Keep each name bound to the same immutable image while held. Use Cocoon v0.7.2 or newer and set `SANDBOX_ADDR` and `SANDBOX_API_TOKEN` for the client. -A sandbox token alone cannot manage volumes. +A sandbox token alone cannot manage volumes. For a complete sleep and wake sequence, +see [the external controller example](volume-controller.md). + +A configured node root token can [register backing paths](sandboxd-api.md#put-v1volumesname) +without editing static config. Registration persists only node-local catalog +metadata and does not manage disk files. An empty runtime `tenants` list allows +only root to initiate attachment, including attachment to a tenant-owned claim. +Keep `api_token` configured. List tenants explicitly to grant self-service; +that permission also allows reattachment to another claim after detach. For a sleep and wake workflow, an +external controller owns quiescence and unmount before detach, a reservation +through sleep, and access gating until wake, attach, and mount complete. That +gate covers API and preview access on every node that can wake the sandbox. +Wake establishes only silkd readiness. The [controller example](volume-controller.md) +has SDK and HTTP contract tests. The controller still supplies the reservation, +guest operations, and access gate. Live changes support mounted and attach-only +entries in read-only or writable mode. Pause application reads before replacement. Close files, mmap mappings, cwd, and bind mount references on the old disk. Replacement unmounts the old disk @@ -58,9 +73,9 @@ with client.new("rt:24.04", ttl_seconds=900) as sb: A successful run reads `v1`, then `v2`, and preserves every identity assertion. The repeated full target verifies that a settled mount can be retained. -This example starts without other volumes. If your sandbox has writable or -attach-only volumes, retain them in a full PUT or use PATCH to replace only -the dataset names. `None`, a nil Go slice, and a missing `volumes` field do not +This example starts without other volumes. A full PUT replaces the entire +desired set, including writable and attach-only entries. Retain other entries +in that list, or use PATCH to replace only the dataset names. `None`, a nil Go slice, and a missing `volumes` field do not mean an empty target. Go uses the same owning handle and client API token. @@ -91,3 +106,51 @@ admission holds. Detailed errors survive only within one daemon lifetime. If the VMM exits, a sandbox holding volumes is release-only. Release it and create a new sandbox; neither PUT with an empty target nor daemon restart restores the original processes. + +## Writable and attach-only entries + +Use `mode: "rw"` for a catalog entry that permits writing. Set per-entry +`attach_only: true` when the caller manages the device. PUT and PATCH accept +all four modes in a mixed list: + +```json +{"volumes":[{"name":"dataset"},{"name":"scratch","mode":"rw"},{"name":"raw-data","attach_only":true},{"name":"raw-work","mode":"rw","attach_only":true}]} +``` + +Omitting a mount still selects `/volumes/` for a new mounted entry. An +attach-only entry cannot supply a nonempty mount. HTTP responses keep the +legacy `{name, mount?, mode?}` fields and omit `attach_only`. The current SDKs +infer the flag from a mountless response. Echoing that response to the same +sandbox retains the held device-only entry. An omitted or false flag does not +change that held shape. A mount or mode change requires a successful removal +before adding the name again. + +The existing Go claim option `WithVolumesAttachOnly()` and Python `mount=False` +remain claim-wide aliases. The current source SDKs also encode an all-attach-only +initial list with the legacy global flag. They reject mixed mounted and +attach-only initial claims before network I/O. Claim without volumes, as in the +example above, then use the live API for a mixed list. Direct HTTP mixed initial +claims require matching daemons across the serving fleet and redirect targets. +Older claim decoders can silently ignore per-entry flags. See +[version compatibility](sandboxd-api.md#volume-version-compatibility). These +per-entry SDK features are unreleased. Published Python 0.1.19 can still echo a +same-sandbox GET response but cannot construct the new mixed request mappings. + +Before removal, stop application access and close references. For attach-only +devices, unmount the caller-managed filesystem first. sandboxd checks its guest +mount namespace, but cannot prove that raw block users, partitions, or other +mount namespaces are idle. Keep the control-plane reservation across any sleep +and gate access until wake, attach, and mount complete. + +For mounted writable entries, sandboxd clears its dirty marker only after +identifying the expected device at the managed root mount with writable mount +and superblock options, completing a normal unmount, confirming no residual +device mounts, and confirming host detach. The same removal must supply all +that evidence before the old hold is released. +External pre-unmount, an absent device, or a lost result leaves the +marker conservatively in place. Detach can succeed while a subsequent read-only +attach still requires recovery. Attach-only writers have no daemon-managed +marker, including after a guest crash during writes. The caller owns recovery +and must establish consistency before read-only publication. See the [API +contract](sandboxd-api.md#patch-v1sandboxesidvolumes) for failure and retry +semantics. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 31a255cd..8ae30d2b 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -8,7 +8,7 @@ All bodies are JSON. Three token kinds: preview mint) and the tenant-scoped listings/deletes below; everything a tenant creates is stamped with its name. Operator surfaces (the per-id sandbox reads, `PUT /v1/sandboxes/{id}/instance-metadata`, `GET /v1/info`, - `PUT /v1/pools`, `POST/DELETE /v1/drain`, `GET /metrics`, + `PUT /v1/pools`, `PUT/DELETE /v1/volumes/{name}`, `POST/DELETE /v1/drain`, `GET /metrics`, `GET /v1/checkpoints/{id}/blob`) answer a tenant token `403` — authenticated but not authorized; an unknown token stays `401`. On a node with a shared @@ -83,17 +83,17 @@ Auth: `Authorization: Bearer ` (when configured). - `egress` `false` claims with no [egress policy](egress.md), whatever the pool's (or, for a promoted template, its source pool's): the guest's doors are never bound. Omitted or `true` keeps the policy the claim would get -- `volumes` is an ordered list of at most eight unique catalog names. `mount` - defaults to `/volumes/`; a custom value must be absolute and clean, +- `volumes` is an ordered list of at most eight unique catalog names. Unless + `attach_only` is true, `mount` defaults to `/volumes/`; a custom value must be absolute and clean, outside the guest OS tree, unique, and non-nesting within the request. `mode` is `"ro"` (default, omitted) or `"rw"`; `"rw"` requires the catalog entry's `writable: true` (see [deploy](deploy.md#dataset-volumes)) - `volumes_attach_only` (default `false`) attaches every requested volume without mounting it, handing the whole mount contract to the workload. It - requires at least one volume, and rejects any entry carrying a `mount` — - the path is meaningless when the caller mounts the device. Everything below - describes the default, eager behaviour, which is completely unchanged; the - attach-only contract is in [its own section](#attach-only-volumes) + requires at least one volume and rejects entries with a nonempty `mount`. + Per-entry `attach_only: true` selects the same behavior for that entry. + See the [attach-only contract](#attach-only-volumes) and + [version compatibility](#volume-version-compatibility) before using mixed initial claims Success: @@ -129,20 +129,31 @@ and persisted at finalization — `mode` is omitted from the echo for `ro` entries, matching the request shape. sandboxd attaches each disk, waits for the device settle — a serial match under `/sys/block`, then the `/dev/` node itself, since the kernel publishes sysfs before devtmpfs creates it — up -to 2 seconds total, and mounts the filesystem — read-only, unless the entry -requested and was granted `rw` — before returning. A custom mount may shadow +to 2 seconds total, and mounts the filesystem for mounted entries before +returning. An attach-only entry skips that wait and mount. A custom mount may shadow an existing populated guest directory for the claim's life. ### Attach-only volumes -`"volumes_attach_only": true` stops after the attach. The device appears in -the guest, nothing is mounted, and the echoed entries carry no `mount` key: +Set `"attach_only": true` on an individual volume to stop after attachment. +It can share a raw HTTP claim with mounted entries on a matching daemon. +The existing claim-wide +`"volumes_attach_only": true` applies that choice to every entry. A nonempty +`mount` is invalid with either form. The device appears in the guest without a +mount. HTTP response entries omit both `mount` and `attach_only`: ```json {"id": "sb_…", "token": "…", "volumes": [{"name": "imagenet"}, {"name": "scratch-db", "mode": "rw"}]} ``` +Claim volume entries use this legacy shape in every HTTP response, including +nested sandbox summaries and partial failure views. The current SDKs infer attach-only from +a mountless response. Returning that response to the same sandbox preserves +the held attachment shape. Using it in a fresh claim requires an explicit +`attach_only: true` or the claim-wide flag. New entries without either flag +use the default mount. + Find each device by its virtio serial, which is the catalog name — poll this, not a one-shot lookup: the claim can return before the guest enumerates the device (typically within ~100ms, never guaranteed), and the `/dev/` node @@ -173,27 +184,26 @@ mounts it, so the guarantee does not depend on your mount flags. The whole consistency contract moves to the caller with the mount: -- sandboxd writes no dirty marker for an attach-only `rw` claim and clears - none, because it cannot verify your unmount and therefore promises nothing. - The flush is yours too: release runs no unmount and no `sync`, so whatever - the workload left unsynced is discarded with the VM. Releasing without a - clean unmount of your own leaves the image exactly as a filesystem crash - would. The next *eager* `ro` claim then fails at mount time with a 500 — - loud and attributable to the claim that skipped its unmount — instead of - the marker's 409. Recovery is any `rw` cycle that replays the journal. +- sandboxd writes and clears no dirty marker for attach-only `rw`. A guest + crash while writing also leaves no daemon-managed crash-dirty record. + Release runs no unmount and no `sync` for these entries. The caller must + stop writes, flush, and unmount before detach or release, and owns recovery + after a crash. A later `ro` claim can be admitted without a marker and may + mount an inconsistent filesystem. Mount failure is not guaranteed. Establish + filesystem consistency before publishing the image for read-only use. - a marker left by an earlier *eager* `rw` crash is not cleared by attach-only cycles, and still refuses eager `ro` claims with 409 until a `rw` claim releases cleanly. - writes straight to the block device bypass the filesystem journal entirely and are outside marker protection in either mode. -What still protects other claims is unchanged: admission excludes an -attach-only `rw` claim against every other claim of the name (and readers -against a writer), an attach-only `ro` claim of a marker-bearing image is -refused with 409, and a claim with volumes still refuses checkpoint, fork and -hibernate. An attach-only claim costs one disk attach per volume — no device -settle, no mount round-trip — and release costs nothing at all: removing the -VM closes the devices. +Admission excludes an attach-only `rw` claim against every other claim of the +name, and excludes a writer while readers hold the name. An attach-only `ro` +claim of a marker-bearing image is refused with 409. Capture remains refused +while volume holds or recovery references remain. An attach-only claim performs +one disk attach per volume without waiting for guest enumeration or mounting +it. Release removes the VM and +closes its devices without unmounting caller-managed filesystems. Redirects (mutually exclusive with the fields above) name peers to retry at — sent on a warm miss with warm peers, when the node lacks a golden for @@ -221,7 +231,7 @@ an unknown and a forbidden volume return the same error text. Errors: 400 unknown template axis, metadata over a bound (the message names it), an unknown `on_expire`, invalid/duplicate volumes, -`volumes_attach_only` with no volumes or with an entry carrying a `mount`, +`volumes_attach_only` with no volumes or with an entry carrying a nonempty `mount`, `mode: "rw"` against a non-writable entry, or a volume that is unknown or forbidden (the latter two are deliberately indistinguishable), or @@ -235,6 +245,27 @@ release before `ro` claims resume; see calling tenant at its own `max_claims`, or the node draining; 500 provisioning failed. +### Volume version compatibility + +Per-entry `attach_only` describes the current source API. Older claim decoders +can ignore that field and mount the disk instead. Before sending a mixed +initial claim over HTTP, upgrade every serving node, including redirect +targets. Do not retry a rejected live request after dropping `attach_only`. +An older live endpoint with strict request decoding rejects the new field +before changing devices. + +The current source Go and Python SDKs send all-attach-only initial claims using +the existing `volumes_attach_only` flag without per-entry flags. They reject +mixed mounted and attach-only initial claims locally, before network I/O. +That restriction can be removed when the minimum supported server version +honors per-entry `attach_only` on initial claims at every possible placement +and redirect target. Until then, the SDK cannot know that a redirected claim +will preserve the mixed mounting choice. +Claim without volumes, then use PUT or PATCH for a mixed live list. These SDK +changes are unreleased. The published Python 0.1.19 SDK accepts only `name`, +`mount`, and `mode` in request mappings. HTTP responses retain those fields, +so its same-sandbox GET-to-PUT workflow remains supported. + ## GET /v1/volumes Auth: node API token (root or tenant). Lists the fleet catalog entries the @@ -255,9 +286,137 @@ list; the field is emitted (as `true`) only for a writable entry this node declares and omitted otherwise (peer-only rows never carry it), so a read-only entry's response is byte-identical to v1. +## PUT /v1/volumes/{name} + +Auth: the configured root node API token. With no configured `api_token`, +registration and unregistration return 403. A tenant token returns 403. +Sandbox, unknown, and missing tokens return 401 when `api_token` is configured. + +Registers a backing path on this node. The URL supplies the name; +the body has no `name` field. Unknown fields, duplicate fields, and trailing +JSON return 400. + +```json +{"path": "/srv/datasets/data.img", "directio": "off", + "writable": false, "tenants": ["acme"]} +``` + +- `name` matches `[a-z][a-z0-9_-]{0,19}` and cannot start with `cocoon-`. +- `path` uses the same absolute-path validation as static volume config. + Registration does not open the backing or check its type or availability. + A missing backing retains its registration but cannot be attached until available. + Registration success does not establish that Cocoon supports the backing type. + Operators must keep the registered path and target stable. + Backing storage must be separate from daemon cleanup directories, including + `/readers`, `/goldens`, and `/fork-*`. + Backing paths must not resolve into cleanup directories. + Cleanup directories must not resolve into backing storage. + Startup and staging cleanup do not consult volume registrations and can + delete backing files placed in those locations. +- `directio` is `on`, `off`, or `auto`, and defaults to `off`. +- `writable` defaults to `false` and controls writable claim admission. +- `tenants` grants the named tenants self-service access. Omitted or empty + makes a runtime entry operator-only. Root can attach it to a tenant-owned + claim; that tenant cannot add it to another claim. Static config entries + retain their existing public default for an empty list. + Names use the tenant API name syntax. A valid name may be registered before + that tenant exists; it grants no access until the tenant is created. + +Success is 204 with an empty body. Repeating the same name, cleaned path, and +normalized registration fields succeeds. A different binding or different +registration fields return 409. Static and runtime registrations cannot share +a name or a cleaned backing path. Invalid registration fields return 400. +Violations of the catalog ownership, mode, directory, or file-type policy return 409 +without `Retry-After`, before accepting a new write. This refusal does not block +claims using the existing catalog. The storage configuration must be repaired +before that mutation can succeed. These checks cover metadata ownership, +permissions, and path types, not every storage failure. Read-only filesystems, +ENOSPC, immutable attributes, or access errors during the actual write still +enter persistence recovery and can block volume admission until storage is repaired. + +An uncertain write returns 503 with `Retry-After`. The requested registration +may still take effect through background retry or a later PUT or DELETE, +including a request for another name. A 503 can also reject a request before +acceptance, so it does not promise that the request will take effect. After +repairing storage failures, retry the exact request to establish completion. +Cancellation is not a rollback once a write has been accepted. +GET retains its existing projection without host paths or tenant access lists. +Registered entries support the live volume modes described below. + +Only registration metadata is persisted locally in +`data_dir/runtime-volumes.json`, which loads before claim recovery. The record +is a root-owned regular file with mode 0600. The data directory and every +ancestor must be root-owned directories, not symlinks. Group-writable or +world-writable directories must have the sticky bit set. +Runtime mutations require sandboxd to run as OS root. The backing file does +not require root ownership. Registration creates, grows, and deletes no disks. +An OS-nonroot daemon refuses runtime mutations with 409 and no `Retry-After`. +Malformed or conflicting metadata prevents startup. A temporarily missing +backing file keeps its registration and refuses new attachment. +Catalog replacement and synchronization are independent of `sync_claims`, which +controls claim snapshot synchronization. Process-restart checks do not prove +power-loss recovery. The catalog is node-local with one sandboxd writer. +Startup reads only `runtime-volumes.json`. Crash leftovers named +`runtime-volumes.json.*.tmp` are neither loaded nor promoted. An operator can +remove those leftovers after stopping sandboxd and preserving any needed +evidence. + +## DELETE /v1/volumes/{name} + +Auth and name validation match PUT. Removes only the runtime registration and +never deletes or changes the backing file. Success, including an absent, +unreferenced runtime name, is 204 with an empty body. Static registrations and +names referenced by a claim, an in-flight operation, cleanup, or an +unacknowledged recovery snapshot return 409. Catalog persistence uncertainty +returns 503 with `Retry-After` and has the same asynchronous completion +semantics as PUT. A failed DELETE may therefore remove the registration later. +An empty GET volume view alone does not establish that a recovery reference +has retired. + +An external controller owns quiescence and guest unmount before detach. It +confirms successful detach and recovery bookkeeping before hibernation, and +keeps its own volume reservation while the sandbox sleeps. sandboxd holds no +reservation after detach and reference retirement, so root can attach that +volume elsewhere during sleep. For controller-only attachment, leave the runtime +entry's `tenants` empty and keep `api_token` configured. A nonempty ACL allows those tenants to attach the +volume themselves after detach; it does not reserve the volume for one claim. +Removing `api_token` restores the daemon's unauthenticated root-scope mode, so +operator-only admission then provides no network authentication. The controller +gates every access path, including the API and preview listeners on all nodes +that can serve or wake the sandbox, until wake, attach, and mount complete. +Wake establishes only silkd readiness. +It does not restore volume attachments or application readiness. Controller +orchestration is external. The [controller example](volume-controller.md) +shows the sequence, the pool replication and archive settings, and the +reservation and access-gate contracts it requires. A change of owning node +requires the controller to fence the old node before attaching the same backing +on a new node, even when the claim ID is unchanged. +Downgrading to a daemon without runtime registration requires draining runtime references or supplying equivalent +static volume config. Pending live volume changes must finish before downgrade. + +Earlier development builds treated empty runtime ACLs as public. Loading those +records now restricts new admission to root without discarding existing claims +or blocking their cleanup. Before downgrading to a build with the old ACL rule, +drain and unregister operator-only entries. Leaving their registrations in place +would make them public again; draining claims alone does not prevent that. + +An unresolved stale VM keeps volume admission and catalog changes fenced while +cleanup retries continue. Inspect success alone does not prove a complete disk +inventory. A present VM with omitted disks, or an active creating owner, retains +that fence until a complete running inventory or confirmed removal resolves it. +Cocoon v0.7.3 omits the disk list when its inventory query fails, and does not +collect it for stopped VMs. An empty list is therefore not proof of absence. +This availability limit also affects static read-only admission. Allowing readers +would conflict with an unknown writer. Cleanup retries remain enabled. +A warm VM name is not evidence that no volume is attached; attachment can +succeed before claim finalization fails. Known inventories keep holds on their +reported names. Volume-free claims continue to work. Path uniqueness uses +cleaned path strings. The operator coordinates symlink and hardlink aliases +and cross-node ownership. + ## GET /v1/sandboxes/{id}/volumes -Returns the persisted desired set (`volumes_want`), confirmed applied set +Returns the current desired set (`volumes_want`), confirmed applied set (`volumes`), per-volume `states`, and `converging`. Auth requires a client API token. Root can access every sandbox; a tenant can access only its own. Sandbox tokens are rejected when authentication is enabled. The owning node @@ -277,7 +436,8 @@ when the volume is in both sets, `pending` when desired but not yet confirmed, or `detaching` when applied but no longer desired. An interrupted device call can leave the actual guest state different from the confirmed set. A failed PUT or PATCH adds a top-level `error` naming the refusal, and the affected -state may carry its own `error`. `converging` is true only while a request worker is running. +state may carry its own `error`. `converging` is true while a request worker +is running or a recovery-only reference awaits an acknowledged claim write. `pending` with `converging: false` requires an explicit retry; there is no background convergence. GET retains the last safe per-volume error for the current daemon lifetime. After daemon restart, unfinished states report an @@ -287,22 +447,32 @@ The sandbox index exposes both sets for operator inspection. ## PUT /v1/sandboxes/{id}/volumes Replaces the full desired set and converges synchronously. Auth matches GET. -The owner tenant's catalog access list applies to newly admitted names, including -for root. Retaining, repairing, or removing held names does not reauthorize +For static entries and runtime entries with a nonempty ACL, the owner tenant's +access list applies to newly admitted names, including for root. An empty runtime +ACL instead requires the root caller, which can attach it to a tenant-owned claim. +Retaining, repairing, or removing held names does not reauthorize them. A new image must exist on the owning node. The request must include a non-null `volumes` -array. An explicit empty array removes all mutable entries. +array. An explicit empty array removes all entries. ```json {"volumes": [{"name": "imagenet", "mount": "/datasets/training"}]} ``` -Only mounted read-only catalog entries can change live (`mode` omitted or -`ro`). A PUT must retain every existing writable or attach-only entry unchanged. +Mounted read-only, mounted writable, attach-only read-only, and attach-only +writable entries can be added or removed live. `mode` defaults to read-only; +`rw` requires a writable catalog entry and excludes other readers and writers. +Set `attach_only: true` on an entry to attach its device without mounting it. +A nonempty `mount` with that flag is invalid. The legacy claim-wide +`volumes_attach_only` flag still applies to every entry in a new claim. The claim-time name, mount, and eight-volume limits apply to the desired set. -An omitted mount defaults to `/volumes/`, except that a held attach-only -entry is retained by sending it as the claim returned it, without `mount`. To -change an existing name's -mount or mode, first remove that name and wait for successful convergence. +A new entry without `attach_only: true` and with omitted `mount` defaults to +`/volumes/`. HTTP responses omit `attach_only` and omit `mount` for +device-only entries. GET's desired list can be echoed to the same sandbox. +For an existing held mountless entry, an empty or omitted mount with an omitted +or false `attach_only` preserves the attach-only shape. Explicit false is +equivalent to omission here. It does not request a change to a managed mount. +To change an existing name's nonempty mount, mode, or mounted shape, first +remove that name and wait for successful convergence. An identical target can be retried. Claims with `on_expire: archive` must change that policy before adding a volume. @@ -326,13 +496,34 @@ unchanged. An empty patch retries the current desired set. ``` Adding an identical entry or removing an absent valid name is harmless. -Duplicate names, a name in both arrays, or changes to writable or attach-only -entries return 400. Removal refuses busy or foreign mounts and uses neither +Duplicate names, a name in both arrays, or changes to an existing held entry's +mount, mode, or attach-only shape return 400. Removal refuses busy or foreign mounts and uses neither lazy nor forced unmount. A foreign mount, another bind mount, or a completed ordinary unmount with nonzero exit returns 409 with the guest mount path and a safe per-volume reason. Guest transport failures remain server errors; timeouts return 504 and cancellation returns 408. +For attach-only entries, the caller must stop writes and unmount before detach. +The daemon refuses mounts it can observe for the device; it does not unmount a +caller-managed filesystem. That probe does not establish that raw block users, +partitions, or other mount namespaces are idle. The controller owns those +conditions. + +Mounted writable entries receive a dirty marker before attachment. Live removal +clears it only after sandboxd identifies the expected device at its managed root +mount and observes writable options for both that mount and its superblock. +The same removal must complete a normal unmount, confirm no remaining device +mounts, and receive successful host detach. Marker cleanup finishes before +writer admission is released. Missing, malformed, or read-only mount evidence +cannot prove a clean unmount. +An already absent device, an externally unmounted filesystem, or uncertain +host removal supplies no clean-unmount evidence. Detach can complete while the +marker remains, and a later read-only request can return 409 requiring recovery. +A marker-file removal failure is logged and retains the conservative marker. +Attach-only writers have no daemon-managed dirty marker, even after a guest +crash during writes. Their caller owns filesystem recovery and safe read-only +publication. + ### Partial failure and recovery PUT and PATCH return the GET response shape on success. A convergence error @@ -346,6 +537,39 @@ retrying. Retry the same PUT to converge, or send a new target to abandon unfinished additions. Concurrent requests serialize per sandbox without blocking renewals or new exec, file, and port connections. +Adding a live RW volume requires a pool with `replicate_seconds: 0`. A +replicating pool returns 409 `live writable volumes require a non-replicating +pool` before recording the new target, reserving the volume, or changing devices. +This restriction applies to live additions, including attach-only RW. An RW +volume attached in the initial claim remains supported, and existing RW volumes +can be retained or removed while replication is configured. RO live changes +remain available. The effective pool is the claim's policy source when it has +one. Changing `replicate_seconds` requires a daemon restart. + +A non-replicating pool can still have an existing or pending replica after a +restart. Live RW additions return 503 with `Retry-After` until that claim's +replicas and recovery checkpoint references have retired. The background scan +retires existing replicas; keep the controller's reservation and access gate +closed, then retry the request. A failed claim write retains the affected +checkpoint reference until a successful snapshot acknowledges its removal. +Unrelated claims' writes and checkpoint references do not refuse this +attachment. Without `archive_failover`, live RW additions return 409 while the +claim retains replica or archive intent fields. See +[disabling archive failover](deploy.md#disabling-archive-failover) for transition +checks, local archive exceptions, and recovery risks. + +A retry of an admitted desired volume completes that operation without creating +a new admission. Its volume hold continues to prevent replica publication. +After a request ends, `converging` is false unless ordinary recovery-reference +retirement remains active; no background worker attaches the desired disk. + +Consumed replica rows, unpublished checkpoints and queued deletion markers are +retained while the affected checkpoint remains in a current claim projection, +an acknowledged snapshot, or an attempted replacement whose result is uncertain. +A later acknowledged snapshot retires removed references and permits cleanup. +Checkpoint deletion waits only for actual references to that checkpoint, not +for a node-wide pause in claim writes. + Before applying a target, sandboxd journals `VolumesWant` beside `Volumes`, storing it only while it differs from `Volumes`. A superseding request may first clean up an older desired-only attachment while @@ -356,8 +580,13 @@ teardown; a teardown keeps them until the VM is confirmed gone. Failed or uncertain operations retain their holds until successful cleanup. Startup reserves those holds without replaying attachments. Release cancels convergence and transfers the held union to teardown without waiting for a blocked device call. Capture operations (hibernate, fork, checkpoint, and -promote) are refused while either set is nonempty or a convergence request -still owns admitted volume holds. +promote) are refused while either set is nonempty, a convergence request +still owns admitted volume holds, or unacknowledged recovery references remain. + +A successful removal response confirms device removal and retirement of the +old recovery reference. After an error or timeout, GET helps select the next +retry, but an empty view alone does not prove that retirement completed. +Retry the removal successfully before hibernating. Device calls honor cancellation and have a 30-second ceiling per volume; a caller that disconnects cancels the run, leaving interrupted states to retry. Journal writes diff --git a/docs/sdk-python.md b/docs/sdk-python.md index d726d366..87ecfe15 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -106,16 +106,17 @@ sb = client.new("ghcr.io/cocoonstack/sandbox/rt:24.04", |---|---|---|---| | `net` | `"none"`, `"egress"` | `"none"` | Cloud Hypervisor network shape: `none` disables the NIC and uses vsock-only I/O; `egress` attaches a bridge/CNI NIC | | `size` | `"small"`, `"medium"`, `"large"`, `"xlarge"`, `"2xlarge"` | `"small"` | resource tier: 1cpu/512M, 2cpu/1G, 4cpu/4G, 4cpu/8G, 8cpu/16G | -| `volumes` | bare names or `{name, mount?, mode?}` mappings | `None` | attach and mount up to eight unique catalog dataset disks; an omitted mount defaults to `/volumes/`; `mode` is `"ro"` (default) or `"rw"` — `"rw"` requires the catalog entry's `writable: true`; accepted by `Client.new` and `Template.new` | +| `volumes` | iterable of bare names or `{name, mount?, mode?, attach_only?}` mappings | `None` | attach up to eight unique catalog dataset disks; mounted entries default to `/volumes/`; `mode` is `"ro"` (default) or `"rw"`, which requires the catalog entry's `writable: true`; `Client.new` and `Template.new` accept all-mounted or all-attach-only lists | | `mount` | bool | `True` | mount every requested volume. `False` attaches the devices and leaves the mounting to the workload; a mapping carrying `mount` is then a `TypeError` | | `ttl_seconds` | int | server default 5m | sandbox TTL, server-capped at 24h. The node reaps the sandbox after the TTL even if the client vanishes | | `deadline` | `time.monotonic()` value | `None` | keyword-only wall clock over the whole claim, redirect targets included: every request is bounded by whatever is left and a spent deadline raises `TimeoutError`. Without it each redirect candidate gets a fresh `timeout`, so a cluster with unreachable peers can cost a multiple of it. Also on `Template.new` and `Checkpoint.new` | `new` returns when the sandbox's silkd answers: a warm hit is milliseconds, a cold key can take the full boot. A volume claim may consume an ordinary warm -VM and returns only after every requested disk is mounted; `sb.volumes` contains -dictionaries with the finalized name, effective mount, and (for `rw` entries) -mode. Custom mounts must be absolute and clean, stay outside the guest OS +VM and returns after each mounted entry is ready. Attach-only entries skip +mounting and may still be awaiting guest device enumeration. `sb.volumes` +contains the finalized name, effective mount, mode, and attachment choice. +Custom mounts must be absolute and clean, stay outside the guest OS tree, and cannot duplicate or nest. The handle exposes `sb.id`, `sb.token`, `sb.owner`, `sb.deadline`, and `sb.from_checkpoint` (the lineage edge when @@ -123,20 +124,46 @@ branched). `sb.template_digest` is the exact content identity when the claim cloned a promoted template; it is empty for other sources. `Sandbox` is a context manager; `sb.close()` releases it (releasing one already gone is not an error — double-release and reap races stay silent). -Volume sandboxes cannot hibernate, fork, checkpoint, or promote. Checkpoint +A sandbox cannot hibernate, fork, checkpoint, or promote while volume holds or +unacknowledged recovery references remain. Checkpoint branches do not accept volumes of either mode in this version. -`mount=False` on `client.new` or `template.new` claims the same volumes -without mounting them: the dictionaries in `sb.volumes` carry no `mount` key, -and the workload finds each device by polling `/sys/block/*/serial` for the -catalog name — not guaranteed present when the claim returns, typically -within ~100ms — then confirms the `/dev/` node itself exists before -mounting. Everything above describes the default (`mount=True`) and is -unchanged by it. What changes is that the mount and its consistency are -entirely yours: sandboxd writes and clears no dirty marker for an attach-only -`rw` claim, because it cannot verify your unmount, so releasing without -unmounting cleanly leaves the image as a crash would — see -[sandboxd-api](sandboxd-api.md#attach-only-volumes) for the full contract. +`volumes` accepts lists, tuples, generators, and sets of names. The SDK consumes +an iterable once. A string, byte string, or single top-level mapping is not a +volume list. + +`mount=False` on `client.new` or `template.new` attaches every volume without +mounting it. Setting `attach_only: True` on every entry has the same effect. +Both forms send the legacy claim-wide flag and omit per-entry flags. Mixed +mounted and attach-only initial claims fail locally before an HTTP request. +Claim without volumes, then use the live API for a mixed list: + +```python +with client.new("rt:24.04") as sb: + view = sb.set_volumes([ + {"name": "dataset"}, + {"name": "scratch", "mode": "rw"}, + {"name": "raw-data", "attach_only": True}, + {"name": "raw-work", "mode": "rw", "attach_only": True}, + ]) +``` + +This avoids older claim decoders silently mounting an entry whose new flag +they ignore. See [version compatibility](sandboxd-api.md#volume-version-compatibility) +before sending mixed initial claims directly over HTTP. The per-entry SDK +features describe the current unreleased source. The published Python +0.1.19 SDK does not accept `attach_only` in request mappings. + +The current SDK exposes attach-only response entries with `attach_only: True` +and no `mount` key. Poll `/sys/block/*/serial` for the catalog name, then +confirm that the `/dev/` node exists before mounting. The claim may return +before either appears. sandboxd writes and clears no dirty marker for +attach-only `rw`, including a guest crash during writes. The caller owns +filesystem consistency, recovery, and safe read-only publication. See the +[attach-only contract](sandboxd-api.md#attach-only-volumes). +For a root client managing a tenant claim, see the +[external volume controller example](volume-controller.md). It keeps the +controller reservation and access gate across detach, hibernate, and wake. The caller-visible constraints are deliberate: volume claims may consume a warm VM, remain non-capturable, mount read-only by default, and require Cloud @@ -161,18 +188,29 @@ peer advertising both the template and every volume; when that intersection is empty, one volume holder may self-verify access to a shared template store before provisioning. -## Changing read-only datasets +## Changing volumes Use `sb.get_volumes()`, `sb.set_volumes([...])`, and `sb.patch_volumes(add=[...], remove=[...])` on the same running sandbox. These methods use the client's API token on the owning node; a sandbox token cannot -manage datasets. `set_volumes([])` clears mutable entries; `None` is rejected. -PUT must retain writable and attach-only entries. PATCH retains omitted names. +manage volumes. Live changes support mounted and attach-only entries in either +mode, including mixed lists. `set_volumes([])` removes all entries; `None` is +rejected. PATCH retains omitted names. To change a held name's mount, mode, or +mounted shape, remove it successfully before adding it again. Each method takes `timeout=` (default: the client timeout, 120 s); a large replacement can need longer, and a call that times out cancels the server run, so retry the same target. Responses include `volumes_want`, `volumes`, `states`, and `converging`. +The HTTP entries keep the legacy `{name, mount?, mode?}` shape and omit +`attach_only`. The current SDK infers `attach_only: True` from mountless +response entries, including partial failure views. Published Python 0.1.19 +can still echo a response to the same sandbox with +`sb.set_volumes(sb.get_volumes()["volumes_want"])`. A new entry without a mount +uses the default mount. A held mountless entry retains attach-only semantics +when echoed with an omitted or false flag. Explicit nonempty mount or mode +changes still require removal first. + `VolumePhase` and `VOLUME_ATTACHED`, `VOLUME_PENDING`, and `VOLUME_DETACHING` name the three states. A pending state with `converging=False` needs an explicit retry. `VolumeError`, exported from `cocoonsandbox` and `cocoonsandbox.errors`, diff --git a/docs/sdk.md b/docs/sdk.md index 2d66f2aa..670f549a 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -210,14 +210,15 @@ defer sb.Close() |---|---|---|---| | `WithNetwork(n)` | `NetNone`, `NetEgress` | `NetNone` | Cloud Hypervisor network shape: `NetNone` disables the NIC and uses vsock-only I/O; `NetEgress` attaches a bridge/CNI NIC | | `WithSize(s)` | `Small`, `Medium`, `Large`, `XLarge`, `XXLarge` | `Small` | resource tier: 1cpu/512M, 2cpu/1G, 4cpu/4G, 4cpu/8G, 8cpu/16G | -| `WithVolumes(volumes...)` | `Volume{Name, Mount?, Mode?}` entries | none | attach and mount up to eight unique catalog dataset disks; `Mount` defaults to `/volumes/`; `Mode` is `"ro"` (default) or `"rw"` — `"rw"` requires the catalog entry's `writable: true`; supported by `Client.New` and `Template.New` | +| `WithVolumes(volumes...)` | keyed `Volume` entries with `Name`, optional `Mount`, `Mode`, and `AttachOnly` | none | attach up to eight unique catalog dataset disks; mounted entries default to `/volumes/`; `Mode` is `"ro"` (default) or `"rw"`, which requires the catalog entry's `writable: true`; `Client.New` and `Template.New` accept all-mounted or all-attach-only lists | | `WithVolumesAttachOnly()` | — | mount | attach the requested volumes without mounting them; the workload finds each device and owns the mount. Rejects a `Volume.Mount` locally | | `WithTimeout(d)` | duration | server default 5m | sandbox TTL, rounded up to seconds, server-capped at 24h. The node reaps the sandbox after the TTL even if the client vanishes | `New` returns when the sandbox's silkd answers: a warm hit is milliseconds, a cold key can take the full boot. A volume claim may consume an ordinary warm -VM and returns only after every requested disk is mounted; the finalized -name, effective mount, and (for `rw` entries) mode are available in +VM and returns after each mounted entry is ready. Attach-only entries skip +mounting and may still be awaiting guest device enumeration. The finalized +name, effective mount, mode, and attachment choice are available in `Sandbox.Volumes`. Custom mounts must be absolute and clean, stay outside the guest OS tree, and cannot duplicate or nest. `Sandbox.ID`, `Sandbox.Deadline`, and @@ -227,15 +228,16 @@ promoted template; it is empty for other sources. `Owner()` names the owning node, and `Token()` returns the per-sandbox bearer to persist with `ID` for a later `Lookup`; `Close()` releases the sandbox (releasing one already gone is not an error, and `Close` is bounded internally so it stays defer-friendly). -Volume sandboxes cannot hibernate, fork, checkpoint, or promote. Passing +A sandbox cannot hibernate, fork, checkpoint, or promote while volume holds or +unacknowledged recovery references remain. Passing `WithVolumes` or `WithClaimRef` to `Checkpoint.New` returns a local error: checkpoint branches support neither volumes nor a claim reference in this version. `WithMetadata`, `WithArchiveOnExpire` and `WithEnv` apply to `New`, `Template.New`, and `Checkpoint.New`: the labels follow the [claim bounds](sandboxd-api.md#post-v1claim), and a branch never inherits its source's labels or expiry action. -For datasets chosen after the sandbox starts, set the desired read-only mounts -on the same claim: +For volumes chosen after the sandbox starts, set the desired set on the same +claim. Live changes support mounted and attach-only entries in either mode: ```go view, err := sb.SetVolumes(ctx, []sandbox.Volume{ @@ -243,7 +245,7 @@ view, err := sb.SetVolumes(ctx, []sandbox.Volume{ }) view, err = sb.GetVolumes(ctx) view, err = sb.PatchVolumes(ctx, sandbox.VolumePatch{Remove: []string{"imagenet"}}) -// An explicit empty slice clears mutable mounts. A nil slice is rejected. +// An explicit empty slice removes all entries. A nil slice is rejected. view, err = sb.SetVolumes(ctx, []sandbox.Volume{}) ``` @@ -265,6 +267,12 @@ can include an `error`. Methods use the client API token on the owning node and update both public sets from each complete response, including a partial failure. Serialize changes to one handle as with `Renew`. Returned views do not share the handle's slices or dictionaries. +HTTP responses keep the legacy `{name, mount?, mode?}` shape and omit +`attach_only`. The current SDK sets `Volume.AttachOnly` when decoding a +mountless response. An omitted mount in a new request still selects the +default mount. A mountless entry echoed to the same sandbox retains its +existing attach-only meaning, including when `AttachOnly` is false. + Go returns `*sandbox.VolumeError` with a `Response` alongside the returned view. It wraps `*sandbox.APIError`, so `errors.As` can inspect the status. Python raises `VolumeError`, a subclass of `APIError`, with `.response`. A response is absent @@ -274,29 +282,43 @@ the handle retains its last known sets. Retry an interrupted operation with the same full PUT target, or set a new target to recover. `Client.Sandboxes` exposes `VolumesWant` and `Volumes` for inspection with an API token. -A PUT must retain existing writable and attach-only entries unchanged. PATCH -cannot remove or change them. Paused sandboxes are not woken, and images must +A PUT replaces all entries, including writable and attach-only entries. PATCH +retains names it does not mention and can add or remove any of the four modes. +Changing a held name's mount, mode, or mounted shape requires a successful +remove before adding it again. Paused sandboxes are not woken, and images must exist on the owner node. Admission holds cover the union of desired and applied sets. A settled retained entry receives no new attach or mount calls. `converging` -is true only during an active request. Pending or detaching entries with false -require an explicit retry. Daemon restart replaces unfinished diagnostics with -an interrupted message. Capture is refused while either set is nonempty. +is true while a request worker is running or a recovery-only reference awaits +an acknowledged claim write. Pending or detaching entries with false require +an explicit retry. Daemon restart replaces unfinished diagnostics with +an interrupted message. Capture remains refused while either set or +unacknowledged recovery references hold volumes. Release cancels convergence and hands held volumes to teardown. See the [API recovery contract](sandboxd-api.md#partial-failure-and-recovery) and [same-sandbox replacement tutorial](live-volumes.md). A failed sandbox with volume holds is release-only after VMM exit; an empty target cannot recover it. -`WithVolumesAttachOnly()` claims the same volumes without mounting them: the -entries in `Sandbox.Volumes` carry an empty `Mount`, and the workload finds -each device by polling `/sys/block/*/serial` for the catalog name — not -guaranteed present when the claim returns, typically within ~100ms — then -confirms the `/dev/` node itself exists before mounting. Everything -above describes the default and is unchanged by this option. What changes is -that the mount and its consistency are entirely yours: sandboxd writes and -clears no dirty marker for an attach-only `rw` claim, because it cannot verify -your unmount, so releasing without unmounting cleanly leaves the image as a -crash would — see -[sandboxd-api](sandboxd-api.md#attach-only-volumes) for the full contract. +`WithVolumesAttachOnly()` claims the same volumes without mounting them. +Setting `AttachOnly: true` on every entry has the same effect. Both forms use +the legacy claim-wide flag on the wire and omit per-entry flags. Mixed +mounted and attach-only initial claims are rejected locally before any HTTP +request. Claim without volumes, then call `SetVolumes` or `PatchVolumes` with +the mixed list. This prevents older claim decoders from silently mounting an +entry whose new flag they ignore. See [version compatibility](sandboxd-api.md#volume-version-compatibility) +before sending a mixed initial claim directly over HTTP. + +Attach-only entries in `Sandbox.Volumes` have `AttachOnly: true` and an empty +`Mount`. Poll `/sys/block/*/serial` for the catalog name, then confirm that the +`/dev/` node exists before mounting. The claim may return before either +appears. sandboxd writes and clears no dirty marker for attach-only `rw`, even +if the guest crashes while writing. The caller owns filesystem consistency, +recovery, and safe read-only publication. See the [attach-only contract](sandboxd-api.md#attach-only-volumes). + +The added `AttachOnly` field breaks unkeyed Go struct literals. Replace +`sandbox.Volume{"data", "/data", "rw"}` with +`sandbox.Volume{Name: "data", Mount: "/data", Mode: "rw"}`. These per-entry +features describe the current unreleased source, not a new published SDK +version. The caller-visible constraints are deliberate: volume claims may consume a warm VM, remain non-capturable, mount read-only by default, and require Cloud diff --git a/docs/security.md b/docs/security.md index 6c519f49..2861207e 100644 --- a/docs/security.md +++ b/docs/security.md @@ -94,15 +94,32 @@ rather than confirming existence, and operator surfaces answer tenants 403. The volume catalog is an operator-owned data boundary. A volume name and its access list must mean the same thing fleet-wide, although membership is -node-local. An empty entry `tenants` list permits every authenticated scope; a -nonempty list permits only those named tenants, while the root token always -has access. Config load rejects an access-list name that is not a configured -tenant. Claim lookup returns byte-identical errors for an unknown and a -forbidden name, and catalog discovery filters before replying, so a tenant +node-local. An empty static entry `tenants` list permits every authenticated +scope. An empty runtime entry is operator-only. A nonempty list grants the +named tenants self-service access. Root can attach an operator-only runtime +entry to a tenant-owned claim. For other live additions, the claim owner must +satisfy the entry's ACL, even when root sends the request. Access-list names must use valid tenant name syntax. A name that +does not yet identify a tenant grants no access until that tenant exists. +Claim lookup returns byte-identical errors for an unknown and a forbidden name, and catalog discovery filters before replying, so a tenant cannot enumerate restricted entries by probing. Gossip carries names only. Neither gossip, discovery, persisted claims, usage events, nor the sandbox index exposes host image paths or access lists. +Runtime registration and unregistration require a configured root `api_token`; +these endpoints return 403 when the token is unset. The daemon must also run +as OS root to change the root-owned registration file. Registration uses the +same path validation as static configuration. It does not verify backing type +or pin backing identity. Operators must supply a backing supported by Cocoon, +prevent path or symlink target replacement while registered, and coordinate aliases. + +Mounted writable volumes use a dirty sidecar at the configured path plus `.dirty`. +The sidecar directory must be writable and retain the marker across host restarts. +Paths directly under ephemeral directories such as `/dev` do not provide that guarantee. +An operator-controlled symlink in a durable directory keeps the sidecar beside +that symlink, but alternate paths to the same backing still need operator coordination. +Block-device backends are not qualified by registration. Attach-only writable +volumes have no dirty marker and require external consistency checks. + Read-only is integrity protection for the shared image, not confidentiality. Mounting a dataset into an egress-lane sandbox gives that sandbox an export path to every destination its tenant egress policy permits; review the volume access @@ -117,19 +134,48 @@ immutable and publish a new name/path for new content. Changing a running claim's volumes needs the root token or the owning tenant's API token. A sandbox token cannot, so a guest holding its own sandbox token cannot attach another dataset, and every addition passes the same access list -as a claim. +as a claim. An operator-only runtime addition instead checks the root caller. +Already-held references can finish repair or removal without a new ACL grant. +Keep `api_token` configured for controller-only admission; without it the daemon +treats network requests as root. An ACL does not reserve a detached disk across +sleep. The controller must retain that reservation and gate workload access. A writable entry (`writable: true`) adds a channel the read-only model does not have: whichever tenant is permitted to claim it `rw` changes what every other permitted tenant reads next, and any of them can be the writer in turn — a multi-tenant access list on a writable entry is a bidirectional channel between those tenants, not just a shared read. Recommend a writable -entry's `tenants` name exactly one tenant — an empty list permits every -authenticated scope, which for a writable entry means every tenant can write -to every other tenant's next read. A dataset that genuinely needs multiple +entry's `tenants` name exactly one tenant when tenant self-service is intended. +An empty static list permits every tenant to write to other tenants' next reads. +Use an empty runtime list when only the controller should initiate attachment. A dataset that genuinely needs multiple writers needs an out-of-band process for who writes when, which the catalog ACL does not provide. +Live writable and attach-only changes enforce the same catalog ACL and local +reader and writer admission as initial claims. Attach-only callers own quiescence, +unmount, and filesystem consistency. Guest mountinfo checks do not prove the +absence of raw block users, partition users, or users in other mount namespaces. +A mounted writable dirty marker is cleared only when the same removal observes +the expected device at the managed root mount, writable mount and superblock +options, successful normal unmount, no residual device mounts, and confirmed +host detach. Marker cleanup precedes release of the old hold. Missing evidence +leaves the marker in place. + +sandboxd writes and clears no daemon-managed marker for attach-only `rw`, even +when the guest crashes while writing. Local reader and writer exclusion does not prove +the image is consistent after detach or release. A later `ro` claim can be +admitted without a marker. The controller must recover and verify the image +before read-only publication. + +An external controller owns stop-write and unmount before detach, its own +reservation across sleep, and access gating until wake, attach, and mount +complete. Wake establishes silkd readiness, and business traffic may trigger +it. Gate the API and preview listeners on every node that can serve the claim. +Previously issued preview URLs can still wake it. Node-local admission does not +fence a writer on another node. Follow the [controller example](volume-controller.md) +for replication, archiving, ownership changes, and recovery. The controller +supplies the reservation and access gate. See the [API contract](sandboxd-api.md#delete-v1volumesname). + ## Known limitations Facts to plan around, stated so the boundary is honest: diff --git a/docs/volume-controller.md b/docs/volume-controller.md new file mode 100644 index 00000000..84291f72 --- /dev/null +++ b/docs/volume-controller.md @@ -0,0 +1,167 @@ +# Hibernate a sandbox with an external volume controller + +Use the live-volume API to remove a data disk before hibernation and restore +it after wake. This example keeps the same claim and catalog registration. +The controller owns the disk reservation and workload access policy throughout. +It does not add disk lifecycle management to sandboxd. + +## Prepare the claim and controller + +Use a `net: none` claim on its owning node. Egress-lane claims cannot hibernate. +Keep its TTL valid across the sleep interval. Set `replicate_seconds: 0` and +`archive_after_seconds: 0` on its pool, and use `on_expire: "destroy"` on the +claim. Keep that pool configured while the controller owns the claim. If the +pool is removed, the node-wide archive policy becomes the fallback. An +archived or archiving claim refuses renew with 409, so renewing alone cannot +preserve this sleep workflow when archiving is enabled. + +Live RW additions require `replicate_seconds: 0`; a replicating pool returns +409 without recording the new target or reserving the volume. Changing this +setting requires a daemon restart. A non-replicating pool may still hold an old +replica or recovery checkpoint reference, which returns 503 with `Retry-After`. +Keep the reservation and access gate closed and retry attachment after that +claim's retirement has been acknowledged. With archive failover disabled, +retained replica or archive intent fields refuse live RW additions with 409. +Follow [disabling archive failover](deploy.md#disabling-archive-failover) for +transition checks, local archive exceptions, and recovery risks. +A fixed delay does not acknowledge retirement. See +[the live-volume retirement contract](sandboxd-api.md#partial-failure-and-recovery). + +Keep replication disabled throughout this controller workflow. After detach, +the claim no longer holds a volume and could otherwise be replicated again +before wake and reattachment. RW volumes attached in the initial claim remain +supported on replicating pools, but this does not permit later live RW additions. + +Use the current Python SDK with +[per-entry attach-only support](sdk-python.md#changing-volumes), Cocoon v0.7.2 or newer, +and Cloud Hypervisor. + +Prepare and format the backing outside sandboxd. Store it in a dedicated +persistent directory, such as `/srv/volumes`. Keep backing storage separate +from daemon cleanup directories, including `/readers`, +`/goldens`, and `/fork-*`, and prevent aliases between them. +Startup and staging cleanup can delete a backing in those locations even +while it remains registered. + +Register the backing on the claim's node with the configured root token: + +```http +PUT /v1/volumes/home +Authorization: Bearer +Content-Type: application/json + +{"path":"/srv/volumes/home.raw","writable":true,"tenants":[]} +``` + +An empty runtime `tenants` list lets only root initiate attachment. A nonempty +list grants those tenants self-service attachment to their claims, including +reuse after detach. Keep `api_token` configured. The daemon's no-token mode +treats network callers as root. + +Bind a root SDK client to the existing tenant claim. Keep the sandbox token +for guest commands and hibernation; the root client token authorizes volume +changes. + +```python +from cocoonsandbox import Client + +root = Client(owner_addr, api_token=root_node_token) +operator = root.attach(owner_addr, sandbox_id, sandbox_token) +``` + +Supply these application-specific operations before running the functions below: + +- `assert_reserved` verifies that the controller still exclusively reserves this + backing for this claim. The reservation must survive controller restart and + the entire sleep interval. Keep it exclusively held during every step and + renew any lease as needed. An assertion alone is not a lock or lease renewal. + Serialize operations for the same reservation. Verify the owning node too. + An owner change requires fencing the old node before another attachment. +- `close_access` blocks and drains business requests before returning. It must + fail closed. Gate the API and preview listeners on every node that can serve + the claim, including direct ports and guest API access. A previously issued + preview URL can reach any node and wake the claim. Preview tokens have no + per-URL revocation, so stopping new URL issuance is not an access gate. + The controller's own maintenance calls may still reach silkd. +- `quiesce_and_unmount` stops and reaps writers, flushes application state and + filesystem writes, and ordinarily unmounts the volume. It raises if any step + fails. Do not use lazy or forced unmount to turn a busy result into success. +- `mount_and_verify` finds the block device by its catalog serial, mounts the + existing filesystem, and verifies its expected identity and data. It raises + before admitting business traffic if the disk, mount, or checks are wrong. + Start the application behind the closed gate if it needs restarting. +- `open_access` admits business traffic only after those checks have succeeded. + +These callbacks are not supplied by sandboxd. An in-process lock or a local +boolean is not a durable reservation or an access gate for other callers. +The controller must enforce their contracts even when a step fails or its +process restarts. sandboxd holds no reservation for the disk once detach and +reference retirement complete. During sleep, root can attach the same catalog +entry to another claim. The controller must prevent that operation through its +own ownership policy. Root-only attachment does not prevent a guest from +accessing an already attached disk, or prevent the tenant from removing it for +cleanup. + +Use one attach-only writable volume for this example. The controller mounts it +explicitly and owns filesystem consistency after a crash. sandboxd writes no +`.dirty` marker for attach-only RW. Do not substitute an automatically mounted +RW entry without accounting for its [dirty-marker rules](sandboxd-api.md#attach-only-volumes). + +## Controller functions + +Pass the controller operations as `VolumeHooks`. +The functions do not release the reservation. Before the final `open_access` +call, exceptions stop the sequence without attempting to open access or run +another lifecycle operation. If a gate hook fails or has an uncertain result, +that hook and the controller must recover the gate. These functions cannot +prove that an external gate closed or undo a partially completed reopen. + +The [tested Python example](../sdk/python/examples/volume_controller.py) defines +`VolumeHooks`, `hibernate_volume`, and `resume_volume`. Copy that module into +your controller, then supply the callbacks described above. The SDK's regular +pytest suite exercises these functions through HTTP with mocked node replies. +Those tests check request and response compatibility and failure ordering. +They do not prove guest unmount, disk integrity, or an external access gate. + +Call `resume_volume` for initial attachment to an empty running claim, or after +sleep. It wakes through a guest command, then attaches and mounts the disk. +Call `hibernate_volume` after the application has used the mounted disk. +Do not wrap the handle in a context manager that releases the claim when the +controller leaves the sleep function. + +The successful remove response is the completion acknowledgment. Its empty +view is an additional check, not a replacement for that acknowledgment. +The function removes only the named volume. If another volume remains, it +refuses to hibernate rather than silently detaching that disk too. + +## Recover without reopening access early + +If unmount is busy, leave the disk attached and the gate closed. Find and stop +the remaining users before retrying. A guest mount check cannot prove that no +raw-device users, partition users, or other mount namespaces exist. + +If a volume request fails, times out, or loses its response, keep the +reservation and gate. GET can show what remains, but an empty GET view alone +does not acknowledge durable retirement. Retry the same remove target and +require success before calling hibernate. Make the guest cleanup hook safe +for a retry where the filesystem is already unmounted. Do not blindly rerun +application writes, formatting, or an entire lifecycle sequence. + +If the owning node changes, treat the claim as a new placement even if its ID +is unchanged. Keep access closed and fence the old node from the backing before +reattaching on another node. Node-local admission and catalog registration do +not provide a distributed single-writer lock. This example does not implement +failover fencing or replica transfer. + +If hibernate or wake fails, inspect claim state and resume from the confirmed +stage while retaining the reservation. `exec("true")` proves only silkd +readiness. The claim may already have woken because another request reached +it. It still has no data disk until attach, and no data mount until the guest +mount operation succeeds. Any attach, mount, or data-check failure leaves +business traffic blocked. + +After a successful cycle, verify the same claim ID, backing identity, disk +serial, filesystem identity, and expected payload. A new VMM process after +wake is expected. The backing is external to the memory snapshot. Keep the +reservation while the resumed claim uses it; release it only through the +controller's normal ownership process. diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index cf305baa..44cf0f9a 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -764,7 +764,7 @@ func TestAttachOnlyVolumeEndToEnd(t *testing.T) { if err != nil { t.Fatalf("attach-only claim: %v", err) } - if want := []sandbox.Volume{{Name: "scratch", Mode: "rw"}}; !slices.Equal(sb.Volumes, want) { + if want := []sandbox.Volume{{Name: "scratch", Mode: "rw", AttachOnly: true}}; !slices.Equal(sb.Volumes, want) { t.Errorf("claim volumes %+v, want %+v", sb.Volumes, want) } applied := []string{"attach:scratch:rw"} diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 9bbc9113..7b557be2 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -27,6 +27,7 @@ type fakeEngine struct { volumeOps []string volumeVMs map[string]bool guestEnvs map[string]string + mounts map[string]map[string]fakeVolumeMount seq int readerRoot string @@ -34,6 +35,11 @@ type fakeEngine struct { readerAttaches []string } +type fakeVolumeMount struct { + name string + rw bool +} + func newFakeEngine(dir string) *fakeEngine { return &fakeEngine{ real: engine.New("cocoon", nil, nil, false, false, ""), @@ -42,6 +48,7 @@ func newFakeEngine(dir string) *fakeEngine { socks: map[string]string{}, volumeVMs: map[string]bool{}, guestEnvs: map[string]string{}, + mounts: map[string]map[string]fakeVolumeMount{}, } } @@ -66,6 +73,7 @@ func (f *fakeEngine) Remove(_ context.Context, name string) error { _ = l.Close() } delete(f.listeners, name) + delete(f.mounts, f.socks[name]) delete(f.socks, name) if f.volumeVMs[name] { delete(f.volumeVMs, name) @@ -190,10 +198,14 @@ func (f *fakeEngine) DiskAttach(_ context.Context, vmName string, spec engine.Vo return nil } -func (f *fakeEngine) MountVolume(_ context.Context, _, name, mount string, rw bool) error { +func (f *fakeEngine) MountVolume(_ context.Context, sock, name, mount string, rw bool) error { f.mu.Lock() defer f.mu.Unlock() f.volumeOps = append(f.volumeOps, "mount:"+name+":"+mount+":"+volumeMode(rw)) + if f.mounts[sock] == nil { + f.mounts[sock] = map[string]fakeVolumeMount{} + } + f.mounts[sock][mount] = fakeVolumeMount{name: name, rw: rw} return nil } @@ -201,13 +213,18 @@ func (f *fakeEngine) MountLiveVolume(ctx context.Context, sock, name, mount stri return f.MountVolume(ctx, sock, name, mount, rw) } -func (f *fakeEngine) UnmountVolume(_ context.Context, _, mount string) error { +func (f *fakeEngine) UnmountVolume(_ context.Context, sock, mount string) error { f.mu.Lock() defer f.mu.Unlock() - f.volumeOps = append(f.volumeOps, "umount:"+mount) + f.unmountVolumeLocked(sock, mount) return nil } +func (f *fakeEngine) unmountVolumeLocked(sock, mount string) { + f.volumeOps = append(f.volumeOps, "umount:"+mount) + delete(f.mounts[sock], mount) +} + func (f *fakeEngine) SyncGuest(_ context.Context, _ string) error { f.mu.Lock() defer f.mu.Unlock() @@ -215,14 +232,21 @@ func (f *fakeEngine) SyncGuest(_ context.Context, _ string) error { return nil } -func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) error { - if err := f.UnmountVolume(ctx, sock, mount); err != nil { - return err - } +func (f *fakeEngine) DetachVolume(_ context.Context, _, sock, name, mount string) (engine.VolumeDetachResult, error) { f.mu.Lock() defer f.mu.Unlock() + entry, mounted := f.mounts[sock][mount] + normal := mount != "" && mounted && entry.name == name && entry.rw + if mount != "" && mounted && entry.name == name { + f.unmountVolumeLocked(sock, mount) + } + for _, other := range f.mounts[sock] { + if other.name == name { + return engine.VolumeDetachResult{}, &engine.VolumeMountConflict{Message: "volume still has a simulated guest mount"} + } + } f.volumeOps = append(f.volumeOps, "detach:"+name) - return nil + return engine.VolumeDetachResult{NormalUnmount: normal}, nil } func (f *fakeEngine) volumeOpsLog() []string { diff --git a/e2e/volume_catalog_test.go b/e2e/volume_catalog_test.go new file mode 100644 index 00000000..70765d73 --- /dev/null +++ b/e2e/volume_catalog_test.go @@ -0,0 +1,222 @@ +package e2e + +import ( + "encoding/json/v2" + "errors" + "net/http" + "os" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + sandbox "github.com/cocoonstack/sandbox/sdk/go" +) + +func TestRuntimeVolumeControllerAdmissionThroughHTTP(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("runtime catalog writes require OS root") + } + st := startTenantStack(t, "node-token", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, nil) + acme, err := sandbox.Connect(st.addr, sandbox.WithAPIToken("acme-token")) + if err != nil { + t.Fatal(err) + } + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "home.img", "private bytes"), "writable": true}) + if err != nil { + t.Fatal(err) + } + tenantScope := *st + tenantScope.token = "acme-token" + for _, scope := range []struct { + token string + want int + }{{"acme-token", http.StatusForbidden}, {"", http.StatusUnauthorized}, {"node-token", http.StatusNoContent}} { + caller := *st + caller.token = scope.token + if status, body := rawJSON(t, &caller, http.MethodPut, "/v1/volumes/home", string(request)); status != scope.want { + t.Fatalf("registration auth: status=%d want=%d body=%s", status, scope.want, body) + } + } + volumes, err := acme.Volumes(t.Context()) + if err != nil || len(volumes) != 0 { + t.Fatalf("operator-only catalog exposed to tenant: volumes=%v err=%v", volumes, err) + } + volume := sandbox.Volume{Name: "home", Mode: "rw", AttachOnly: true} + if unexpected, claimErr := acme.New(t.Context(), "rt:24.04", sandbox.WithVolumes(volume)); claimErr == nil { + _ = unexpected.Close() + t.Fatal("tenant claimed an operator-only volume") + } else if refusal, ok := errors.AsType[*sandbox.APIError](claimErr); !ok || refusal.Status != http.StatusBadRequest { + t.Fatalf("tenant claim refusal: %v", claimErr) + } + var forbidden []byte + for _, name := range []string{"home", "unknown"} { + status, body := rawJSON(t, &tenantScope, http.MethodPost, "/v1/claim", `{"template":"rt:24.04","volumes":[{"name":"`+name+`"}]}`) + if status != http.StatusBadRequest { + t.Fatalf("claim privacy for %s: status=%d body=%s", name, status, body) + } + if name == "home" { + forbidden = body + } else if string(body) != string(forbidden) { + t.Fatalf("forbidden and unknown responses differ: %s versus %s", forbidden, body) + } + } + first, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer second.Close() + if _, err = first.SetVolumes(t.Context(), []sandbox.Volume{volume}); err == nil { + t.Fatal("tenant PUT attached an operator-only volume") + } + if _, err = first.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err == nil { + t.Fatal("tenant PATCH attached an operator-only volume") + } + operator := st.client.Attach(st.addr, first.ID, "") + if view, attachErr := operator.SetVolumes(t.Context(), []sandbox.Volume{volume}); attachErr != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) { + t.Fatalf("root attachment to tenant claim: view=%+v err=%v", view, attachErr) + } + if view, detachErr := first.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}); detachErr != nil || len(view.Volumes) != 0 { + t.Fatalf("tenant cleanup of root attachment: view=%+v err=%v", view, detachErr) + } + if _, err = second.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err == nil { + t.Fatal("tenant reused the controller volume on another claim") + } +} + +func TestRuntimeVolumeExplicitACLAndStaticPublicThroughHTTP(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("runtime catalog writes require OS root") + } + st := startTenantStack(t, "node-token", []tenants.TenantSpec{ + {Name: "acme", Token: "acme-token"}, {Name: "beta", Token: "beta-token"}, + }, []config.VolumeSpec{{Name: "public", Path: writeVolumeImage(t, "public.img", "shared")}}) + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "private.img", "private"), "tenants": []string{"acme"}}) + if err != nil { + t.Fatal(err) + } + if status, body := rawJSON(t, st, http.MethodPut, "/v1/volumes/private", string(request)); status != http.StatusNoContent { + t.Fatalf("registration: %d %s", status, body) + } + for _, tc := range []struct { + token string + want []string + }{{"acme-token", []string{"private", "public"}}, {"beta-token", []string{"public"}}} { + client, connectErr := sandbox.Connect(st.addr, sandbox.WithAPIToken(tc.token)) + if connectErr != nil { + t.Fatal(connectErr) + } + volumes, listErr := client.Volumes(t.Context()) + var names []string + for _, volume := range volumes { + names = append(names, volume.Name) + } + if listErr != nil || !slices.Equal(names, tc.want) { + t.Fatalf("catalog ACL: names=%v want=%v err=%v", names, tc.want, listErr) + } + claim, claimErr := client.New(t.Context(), "rt:24.04", sandbox.WithVolumes(sandbox.Volume{Name: "public"})) + if claimErr != nil { + t.Fatal(claimErr) + } + defer claim.Close() + _, attachErr := claim.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{{Name: "private"}}}) + if (attachErr == nil) != (tc.token == "acme-token") { + t.Fatalf("explicit ACL admission for %s: %v", tc.token, attachErr) + } + } +} + +func TestRuntimeVolumeDetachHibernateWakeThroughHTTP(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("runtime catalog writes require OS root") + } + st := startTenantStack(t, "node-token", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, nil) + request, err := json.Marshal(map[string]any{"path": writeVolumeImage(t, "home.img", "external bytes"), "writable": true}) + if err != nil { + t.Fatal(err) + } + if status, body := rawJSON(t, st, http.MethodPut, "/v1/volumes/home", string(request)); status != http.StatusNoContent { + t.Fatalf("registration: %d %s", status, body) + } + acme, err := sandbox.Connect(st.addr, sandbox.WithAPIToken("acme-token")) + if err != nil { + t.Fatal(err) + } + claim, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer claim.Close() + other, err := acme.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer other.Close() + operator := st.client.Attach(st.addr, claim.ID, "") + otherOperator := st.client.Attach(st.addr, other.ID, "") + volume := sandbox.Volume{Name: "home", Mode: "rw", AttachOnly: true} + if _, err = operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err != nil { + t.Fatal(err) + } + if err = claim.Hibernate(t.Context()); err == nil { + t.Fatal("captured an attached disk") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusConflict { + t.Fatalf("attached capture refusal: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err == nil { + t.Fatal("root admitted a second writer before detach") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusConflict { + t.Fatalf("second writer refusal: %v", err) + } + view, err := operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || len(view.States) != 0 || view.Converging { + t.Fatalf("detach acknowledgment: %+v %v", view, err) + } + if err = claim.Hibernate(t.Context()); err != nil { + t.Fatal(err) + } + if !hibernated(t, st, claim.ID) { + t.Fatal("detach followed by hibernate did not sleep the claim") + } + registered, listErr := st.client.Volumes(t.Context()) + if listErr != nil || len(registered) != 1 || registered[0].Name != "home" || !registered[0].Writable { + t.Fatalf("registration lost across sleep: %+v %v", registered, listErr) + } + if _, err = other.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err == nil { + t.Fatal("tenant acquired operator-only disk during sleep") + } else if refusal, ok := errors.AsType[*sandbox.APIError](err); !ok || refusal.Status != http.StatusBadRequest { + t.Fatalf("sleeping reservation ACL refusal: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}); err != nil { + t.Fatalf("attachment hold incorrectly survived detach: %v", err) + } + if _, err = otherOperator.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"home"}}); err != nil { + t.Fatal(err) + } + beforeWake := st.eng.volumeOpsLog() + if out, execErr := claim.Exec(t.Context(), "echo", "awake"); execErr != nil || out != "awake\n" { + t.Fatalf("implicit wake: %q %v", out, execErr) + } + if hibernated(t, st, claim.ID) { + t.Fatal("guest request did not wake the original claim") + } + if ops := st.eng.volumeOpsLog(); !slices.Equal(ops, beforeWake) { + t.Fatalf("wake unexpectedly operated on external disks: %v -> %v", beforeWake, ops) + } + view, err = claim.GetVolumes(t.Context()) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || len(view.States) != 0 || view.Converging { + t.Fatalf("wake unexpectedly restored volume references: %+v %v", view, err) + } + view, err = operator.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: []sandbox.Volume{volume}}) + if err != nil || !slices.Equal(view.Volumes, []sandbox.Volume{volume}) || view.Converging { + t.Fatalf("controller reattachment: %+v %v", view, err) + } + if status, body := rawJSON(t, st, http.MethodDelete, "/v1/volumes/home", ""); status != http.StatusConflict { + t.Fatalf("unregister attached restored disk: %d %s", status, body) + } +} diff --git a/e2e/volume_mutation_test.go b/e2e/volume_mutation_test.go index 5fcedef8..759f96b3 100644 --- a/e2e/volume_mutation_test.go +++ b/e2e/volume_mutation_test.go @@ -3,6 +3,7 @@ package e2e import ( "errors" "net/http" + "os" "slices" "strings" "testing" @@ -11,6 +12,65 @@ import ( sandbox "github.com/cocoonstack/sandbox/sdk/go" ) +func TestLiveWritableRemovalClearsMarkerThroughSDK(t *testing.T) { + for _, preUnmount := range []bool{false, true} { + t.Run(map[bool]string{false: "daemon unmount", true: "external unmount"}[preUnmount], func(t *testing.T) { + image := writeVolumeImage(t, "data.img", "original-data") + stack := startTenantStack(t, "node-token", nil, []config.VolumeSpec{{Name: "data", Path: image, Writable: true}}) + writer, err := stack.client.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer writer.Close() + reader, err := stack.client.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer reader.Close() + if _, err = writer.SetVolumes(t.Context(), []sandbox.Volume{{Name: "data", Mode: "rw"}}); err != nil { + t.Fatal(err) + } + if _, err = os.Stat(image + ".dirty"); err != nil { + t.Fatalf("writer has no dirty marker: %v", err) + } + if _, err = reader.SetVolumes(t.Context(), []sandbox.Volume{{Name: "data"}}); err == nil { + t.Fatal("reader admitted while writer was attached") + } + if preUnmount { + stack.eng.mu.Lock() + sock := "" + for guest, mounts := range stack.eng.mounts { + if mounts["/volumes/data"].name == "data" { + sock = guest + break + } + } + stack.eng.mu.Unlock() + if sock == "" { + t.Fatal("writer has no simulated mount") + } + if err = stack.eng.UnmountVolume(t.Context(), sock, "/volumes/data"); err != nil { + t.Fatal(err) + } + } + if _, err = writer.PatchVolumes(t.Context(), sandbox.VolumePatch{Remove: []string{"data"}}); err != nil { + t.Fatal(err) + } + _, markerErr := os.Stat(image + ".dirty") + if preUnmount && markerErr != nil || !preUnmount && !errors.Is(markerErr, os.ErrNotExist) { + t.Fatalf("dirty marker disagrees with current-call proof: preUnmount=%t err=%v", preUnmount, markerErr) + } + _, err = reader.SetVolumes(t.Context(), []sandbox.Volume{{Name: "data"}}) + if (err != nil) != preUnmount { + t.Fatalf("reader admission after detach: preUnmount=%t err=%v", preUnmount, err) + } + if data, readErr := os.ReadFile(image); readErr != nil || string(data) != "original-data" { + t.Fatalf("detach changed backing contents: %q, %v", data, readErr) + } + }) + } +} + func TestLiveVolumesEndToEnd(t *testing.T) { image := writeVolumeImage(t, "dataset.img", "dataset-bytes") stack := startTenantStack(t, "node-token", nil, []config.VolumeSpec{{Name: "dataset", Path: image}}) @@ -52,7 +112,7 @@ func TestRejectedVolumeTargetExplainsRefusalWithoutChangingState(t *testing.T) { t.Fatal(checkErr) } _, err = sb.SetVolumes(t.Context(), []sandbox.Volume{{Name: "dataset"}, {Name: "dataset"}}) - if refusal, ok := errors.AsType[*sandbox.VolumeError](err); !ok || refusal.Status != http.StatusBadRequest || !strings.Contains(refusal.Message, "duplicate volume") { + if refusal, ok := errors.AsType[*sandbox.VolumeError](err); !ok || refusal.Status != http.StatusBadRequest || !strings.Contains(refusal.Message, "duplicates name") { t.Errorf("invalid target explanation: %v", err) } view, err := sb.GetVolumes(t.Context()) @@ -77,3 +137,45 @@ func TestEmptyVolumePatchChangesNothing(t *testing.T) { t.Fatalf("empty patch: %+v err=%v", got, err) } } + +func TestMixedLiveVolumeModesThroughSDK(t *testing.T) { + volumes := []sandbox.Volume{ + {Name: "mounted-ro"}, + {Name: "mounted-rw", Mode: "rw"}, + {Name: "device-ro", AttachOnly: true}, + {Name: "device-rw", Mode: "rw", AttachOnly: true}, + } + var catalog []config.VolumeSpec + for _, volume := range volumes { + catalog = append(catalog, config.VolumeSpec{ + Name: volume.Name, Path: writeVolumeImage(t, volume.Name+".img", "seed"), Writable: volume.Mode == "rw", + }) + } + stack := startTenantStack(t, "node-token", nil, catalog) + sb, err := stack.client.New(t.Context(), "rt:24.04") + if err != nil { + t.Fatal(err) + } + defer sb.Close() + view, err := sb.PatchVolumes(t.Context(), sandbox.VolumePatch{Add: volumes}) + if err != nil { + t.Fatal(err) + } + want := slices.Clone(volumes) + want[0].Mount, want[1].Mount = "/volumes/mounted-ro", "/volumes/mounted-rw" + if !slices.Equal(view.Volumes, want) || !slices.Equal(sb.Volumes, want) { + t.Fatalf("mixed SDK attachment = %+v, handle=%+v, want %+v", view, sb.Volumes, want) + } + view, err = sb.GetVolumes(t.Context()) + if err != nil { + t.Fatal(err) + } + view, err = sb.SetVolumes(t.Context(), view.VolumesWant) + if err != nil || !slices.Equal(view.Volumes, want) || view.Converging { + t.Fatalf("mixed GET-to-PUT = %+v, err=%v", view, err) + } + view, err = sb.SetVolumes(t.Context(), []sandbox.Volume{}) + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || view.Converging { + t.Fatalf("mixed removal = %+v, err=%v", view, err) + } +} diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index fba3efba..a70c4fa3 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -398,9 +398,14 @@ func (c *Config) validateReplication() error { } func (c *Config) validateVolumes() error { - names := make(map[string]struct{}, len(c.Volumes)) - paths := make(map[string]string, len(c.Volumes)) - for _, volume := range c.Volumes { + return ValidateVolumeCatalog(c.Volumes) +} + +// ValidateVolumeCatalog checks the shared static and runtime catalog rules without I/O. +func ValidateVolumeCatalog(volumes []VolumeSpec) error { + names := make(map[string]struct{}, len(volumes)) + paths := make(map[string]string, len(volumes)) + for _, volume := range volumes { if !types.ValidVolumeName(volume.Name) { return fmt.Errorf("volume name %q must match %s and not start with cocoon-", volume.Name, types.VolumeNameRe) } @@ -418,6 +423,11 @@ func (c *Config) validateVolumes() error { if !types.ValidDirectIO(volume.DirectIO) { return fmt.Errorf("volume %q directio must be on, off, or auto, got %q", volume.Name, volume.DirectIO) } + for _, tenant := range volume.Tenants { + if !types.NameRe.MatchString(tenant) { + return fmt.Errorf("volume %q tenant %q must match %s", volume.Name, tenant, types.NameRe) + } + } } return nil } diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 06c4fe3c..dd524fe0 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -471,3 +471,16 @@ func writeConfig(t *testing.T, body string) string { } return path } + +func TestValidateVolumeCatalogTenantNames(t *testing.T) { + for _, tenant := range []string{"", "has space", "_leading", strings.Repeat("a", 64)} { + spec := VolumeSpec{Name: "data", Path: "/images/data", DirectIO: types.DirectIOOff, Tenants: []string{tenant}} + if err := ValidateVolumeCatalog([]VolumeSpec{spec}); err == nil { + t.Errorf("invalid tenant %q accepted", tenant) + } + } + spec := VolumeSpec{Name: "data", Path: "/images/data", DirectIO: types.DirectIOOff, Tenants: []string{"future-tenant", "team/project:v1"}} + if err := ValidateVolumeCatalog([]VolumeSpec{spec}); err != nil { + t.Fatalf("syntactically valid future tenant rejected: %v", err) + } +} diff --git a/sandboxd/engine/volume_live.go b/sandboxd/engine/volume_live.go index e1033a84..e093e10b 100644 --- a/sandboxd/engine/volume_live.go +++ b/sandboxd/engine/volume_live.go @@ -31,7 +31,7 @@ func (e *LiveVolumeUnsupported) Error() string { } type volumeMount struct { - devNum, root, path, options string + devNum, root, path, options, superOptions string } // CheckLiveVolumeSupport rejects runtimes without idempotent live disk attach and detach. @@ -62,39 +62,60 @@ func (e *Engine) MountLiveVolume(ctx context.Context, vsockSocket, name, mount s return e.mountDevice(ctx, vsockSocket, device, name, mount, rw) } +// VolumeDetachResult is usable only with a nil DetachVolume error. +// NormalUnmount requires this call to find the expected device at its root +// mount with writable mount and superblock options, ordinarily unmount it, +// exclude remaining device mounts, and confirm host detach. Absence, an earlier +// external unmount, attach-only requests, read-only state, and errors give no +// proof. It cannot be reused across calls or certify externally managed writers. +type VolumeDetachResult struct { + NormalUnmount bool +} + // DetachVolume unmounts a guest volume before confirming host removal. -func (e *Engine) DetachVolume(ctx context.Context, vmName, vsockSocket, name, mount string) error { +func (e *Engine) DetachVolume(ctx context.Context, vmName, vsockSocket, name, mount string) (VolumeDetachResult, error) { ctx, cancel := context.WithTimeout(ctx, VolumeMutationTimeout) defer cancel() device, found, err := e.findVolumeDevice(ctx, vsockSocket, name) if err != nil { - return err + return VolumeDetachResult{}, err } + normal := false if found { - if err = e.unmountLiveVolume(ctx, vsockSocket, device, mount); err != nil { - return err + if normal, err = e.unmountLiveVolume(ctx, vsockSocket, device, mount); err != nil { + return VolumeDetachResult{}, err } } - _, err = e.run(ctx, "vm", "disk", "detach", vmName, argName, name) - return err + if _, err = e.run(ctx, "vm", "disk", "detach", vmName, argName, name); err != nil { + return VolumeDetachResult{}, err + } + return VolumeDetachResult{NormalUnmount: normal}, nil } -func (e *Engine) unmountLiveVolume(ctx context.Context, vsockSocket, device, mount string) error { +func (e *Engine) unmountLiveVolume(ctx context.Context, vsockSocket, device, mount string) (bool, error) { + if mount == "" { + return false, e.refuseOtherMounts(ctx, vsockSocket, device) + } devNum, mounts, err := e.volumeMounts(ctx, vsockSocket, device) if err != nil { - return err + return false, err } target := e.resolveMount(ctx, vsockSocket, mount, mounts) atTarget := slices.DeleteFunc(mounts, func(m volumeMount) bool { return m.path != target }) + normal := false if len(atTarget) == 1 && atTarget[0].devNum == devNum && atTarget[0].root == "/" { if err = e.UnmountVolume(ctx, vsockSocket, mount); err != nil { if _, ok := errors.AsType[*guestExitError](err); ok { - return &VolumeMountConflict{Message: fmt.Sprintf("cannot unmount %q; close guest references and retry", mount)} + return false, &VolumeMountConflict{Message: fmt.Sprintf("cannot unmount %q; close guest references and retry", mount)} } - return err + return false, err } + normal = writableMountOptions(atTarget[0].options) && writableMountOptions(atTarget[0].superOptions) + } + if err = e.refuseOtherMounts(ctx, vsockSocket, device); err != nil { + return false, err } - return e.refuseOtherMounts(ctx, vsockSocket, device) + return normal, nil } func (e *Engine) volumeMounts(ctx context.Context, vsockSocket, device string) (string, []volumeMount, error) { @@ -108,15 +129,22 @@ func (e *Engine) volumeMounts(ctx context.Context, vsockSocket, device string) ( } var mounts []volumeMount for line := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") { - fields := strings.Fields(line) - if len(fields) < 10 || !slices.Contains(fields[6:], "-") { + mount, filesystem, ok := strings.Cut(line, " - ") + fields := strings.Fields(mount) + fsFields := strings.Split(filesystem, " ") + if !ok || len(fields) < 6 || len(fsFields) != 3 || fsFields[0] == "" || fsFields[2] == "" { return "", nil, errors.New("invalid guest mountinfo") } - mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5]}) + mounts = append(mounts, volumeMount{fields[2], mountUnescaper.Replace(fields[3]), mountUnescaper.Replace(fields[4]), fields[5], fsFields[2]}) } return strings.TrimSpace(string(devNum)), mounts, nil } +func writableMountOptions(options string) bool { + parts := strings.Split(options, ",") + return slices.Contains(parts, "rw") && !slices.Contains(parts, "ro") && !slices.Contains(parts, "") +} + func (e *Engine) volumeMounted(ctx context.Context, vsockSocket, device, mount string, rw bool) (bool, error) { devNum, mounts, err := e.volumeMounts(ctx, vsockSocket, device) if err != nil { diff --git a/sandboxd/engine/volume_live_test.go b/sandboxd/engine/volume_live_test.go index a731f2e1..7a7fecd7 100644 --- a/sandboxd/engine/volume_live_test.go +++ b/sandboxd/engine/volume_live_test.go @@ -54,7 +54,7 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { t.Fatal(err) } } - if err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, tt.mount); err != nil { + if _, err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, tt.mount); err != nil { t.Fatal(err) } fake.mu.Lock() @@ -74,6 +74,48 @@ func TestLiveVolumeConvergesMountAndDetach(t *testing.T) { } } +func TestLiveVolumeAcceptsEmptyMountSource(t *testing.T) { + for _, tt := range []struct { + name, mount string + rw bool + }{ + {name: "mounted read-only", mount: "/data"}, + {name: "mounted writable", mount: "/data", rw: true}, + {name: "attach-only"}, + } { + t.Run(tt.name, func(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + configureVolumeDevices(fake) + spec := liveDiskCLI(t, "") + const unrelated = "501 84 0:62 / /empty-source rw,relatime - tmpfs rw,size=1024k,inode64\n" + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + unrelated) + fake.execHook = func(argv []string) { + switch argv[0] { + case "mount": + mode := mountMode(tt.rw) + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + unrelated + "2 1 8:32 / /data " + mode + " - ext4 /dev/vdc " + mode + "\n") + case "umount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + unrelated) + } + } + fake.mu.Unlock() + e := New("cocoon", nil, nil, false, false, "") + if tt.mount != "" { + if err := e.MountLiveVolume(t.Context(), sock, spec.Name, tt.mount, tt.rw); err != nil { + t.Fatalf("mount with unrelated empty source: %v", err) + } + } + result, err := e.DetachVolume(t.Context(), "sbx-1", sock, spec.Name, tt.mount) + if err != nil || result.NormalUnmount != tt.rw { + t.Fatalf("detach with unrelated empty source: result=%+v err=%v", result, err) + } + }) + } +} + func TestLiveMountRefusesAnOccupiedPathBehindASymlink(t *testing.T) { sock := sockPath(t) fake := serveFakeSilkd(t, sock) @@ -191,7 +233,7 @@ func TestLiveDetachKeepsBusyAndUnobservableDevices(t *testing.T) { fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n3 2 8:32 /sub /data ro - ext4 /dev/vdc ro\n") } fake.mu.Unlock() - err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") + _, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") if err == nil { t.Fatal("detach must fail closed") } @@ -222,7 +264,7 @@ func TestLiveDetachMissingGuestStillChecksHost(t *testing.T) { sock := sockPath(t) serveFakeSilkd(t, sock) spec := liveDiskCLI(t, hostErr) - err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") + _, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data") wantErr := hostErr != "" if (err != nil) != wantErr { t.Fatalf("got %v, want error=%v", err, wantErr) @@ -240,7 +282,7 @@ func TestLiveDetachIgnoresAnotherDeviceAtThePath(t *testing.T) { fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:48 / /data ro - ext4 /dev/vdd ro\n") fake.mu.Unlock() - if err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err != nil { + if _, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, "/data"); err != nil { t.Fatalf("detach of an unmounted device blocked by another device's mount: %v", err) } fake.mu.Lock() @@ -282,3 +324,68 @@ exit 2 func guestChanges(calls [][]string) [][]string { return slices.DeleteFunc(slices.Clone(calls), func(argv []string) bool { return argv[0] == "realpath" }) } + +func TestLiveDetachCleanEvidenceRequiresWritableUnmountAndHostSuccess(t *testing.T) { + for _, mode := range []string{"normal", "readonly", "mount readonly", "contradictory mount", "superblock readonly", "contradictory superblock", "empty option", "unordered writable options", "missing superblock options", "extra superblock fields", "pre-unmounted", "missing device", "host failure", "busy", "residual mount", "attach-only mounted"} { + t.Run(mode, func(t *testing.T) { + sock := sockPath(t) + fake := serveFakeSilkd(t, sock) + if mode != "missing device" { + configureVolumeDevices(fake) + } + hostErr := "" + if mode == "host failure" { + hostErr = "host removal failed after unmount" + } + spec := liveDiskCLI(t, hostErr) + mount := "/data" + fake.mu.Lock() + fake.serial["/sys/block/vdc/dev"] = []byte("8:32\n") + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw - ext4 /dev/vdc rw\n") + switch mode { + case "pre-unmounted", "missing device": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + case "readonly": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc ro\n") + case "mount readonly": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data ro - ext4 /dev/vdc rw\n") + case "contradictory mount": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw,ro - ext4 /dev/vdc rw\n") + case "empty option": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw,,relatime - ext4 /dev/vdc rw\n") + case "superblock readonly": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw - ext4 /dev/vdc ro,errors=remount-ro\n") + case "contradictory superblock": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw - ext4 /dev/vdc rw,ro\n") + case "unordered writable options": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data relatime,rw shared:4 - ext4 /dev/vdc errors=remount-ro,rw\n") + case "missing superblock options": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw shared:4 - ext4 /dev/vdc\n") + case "extra superblock fields": + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo + "2 1 8:32 / /data rw - ext4 /dev/vdc rw extra\n") + case "busy": + fake.execCode = 32 + case "attach-only mounted": + mount = "" + } + fake.execHook = func(argv []string) { + if argv[0] == "umount" && mode != "residual mount" && mode != "busy" { + fake.serial["/proc/self/mountinfo"] = []byte(rootMountInfo) + } + } + fake.mu.Unlock() + result, err := New("cocoon", nil, nil, false, false, "").DetachVolume(t.Context(), "sbx-1", sock, spec.Name, mount) + wantErr := slices.Contains([]string{"host failure", "busy", "residual mount", "attach-only mounted", "missing superblock options", "extra superblock fields"}, mode) + if (err != nil) != wantErr || result.NormalUnmount != (mode == "normal" || mode == "unordered writable options") { + t.Fatalf("mode=%s result=%+v err=%v", mode, result, err) + } + if mode == "attach-only mounted" { + fake.mu.Lock() + defer fake.mu.Unlock() + if len(guestChanges(fake.execCalls)) != 0 { + t.Fatal("attach-only detach unmounted a caller-managed filesystem") + } + } + }) + } +} diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index af26aa17..e6380456 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -280,7 +280,7 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc } func (m *Manager) deleteOrphanArchiveCk(ctx context.Context, ckID string) { - if err := m.deleteArchiveCk(ctx, ckID); err != nil { + if _, err := m.deleteArchiveCk(ctx, ckID); err != nil { log.WithFunc("pool.deleteOrphanArchiveCk").Warnf(ctx, "delete orphaned archive ck %s: %v", ckID, err) } } @@ -351,17 +351,24 @@ func (m *Manager) clearArchiveCk(ckID string) error { return err } -func (m *Manager) deleteArchiveCk(ctx context.Context, ckID string) error { +func (m *Manager) deleteArchiveCk(ctx context.Context, ckID string) (bool, error) { if err := m.markArchiveCk(ckID); err != nil { - return fmt.Errorf("track: %w", err) + return false, fmt.Errorf("track: %w", err) } - if err := m.deleteCkLocked(ctx, ckID); err != nil { - return err + unlock := m.lockRec(ckID) + if _, referenced := m.store.checkpointRefs()[ckID]; referenced { + unlock() + return false, nil + } + err := m.ckpts.Delete(ctx, ckID) + unlock() + if err != nil { + return false, err } if err := m.clearArchiveCk(ckID); err != nil { - return fmt.Errorf("clear: %w", err) + return false, fmt.Errorf("clear: %w", err) } - return nil + return true, nil } func (m *Manager) retryArchiveDeletes(ctx context.Context) { diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index c9ffeee9..468724f9 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -680,11 +680,14 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { m.retryArchiveDeletes(t.Context()) close(retried) }() - waitFor(t, func() bool { - m.recLocksMu.Lock() - defer m.recLocksMu.Unlock() - return m.recRefs[ck] >= 2 - }) + select { + case <-retried: + case <-time.After(3 * time.Second): + t.Fatal("retry did not defer the checkpoint still referenced by the journal") + } + if !ckExists(t, m, ck) { + t.Fatal("retry deleted the checkpoint before wake persistence completed") + } breakStore(t, m) m.store.writeMu.Unlock() locked = false diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index e02b70b2..95eb05cd 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -5,7 +5,6 @@ import ( "encoding/json/v2" "errors" "fmt" - "maps" "os" "path/filepath" "slices" @@ -318,16 +317,16 @@ func (m *Manager) vetoIfHealPending(ckptID string) { func (m *Manager) pinnedArchiveCks() map[string]struct{} { m.mu.Lock() defer m.mu.Unlock() - pinned := maps.Clone(m.pendingCks) + pinned := m.store.checkpointRefs() + if pinned == nil { + pinned = map[string]struct{}{} + } + for id := range m.pendingCks { + pinned[id] = struct{}{} + } for _, sb := range m.claimed { - if sb.ArchiveCk != "" { - pinned[sb.ArchiveCk] = struct{}{} - } - if sb.ReplicaCk != "" { - pinned[sb.ReplicaCk] = struct{}{} - } - if sb.ReplicaPending != "" { - pinned[sb.ReplicaPending] = struct{}{} + for _, id := range checkpointIDs(sb) { + pinned[id] = struct{}{} } } return pinned diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 683074ee..03c2369c 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -371,7 +371,7 @@ func (m *Manager) overQuota(extra int, tenant string) error { // admitVolumes takes one claim's holds; a refusal on either check leaves nothing held. func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types.Volume, error) { applied := appliedVolumes(volumes) - if err := m.admitClaim(tenant, applied); err != nil { + if err := m.admitClaim(tenant, volumes); err != nil { return nil, err } if err := m.confirmVolumesClean(volumes); err != nil { @@ -382,13 +382,13 @@ func (m *Manager) admitVolumes(tenant string, volumes []resolvedVolume) ([]types } // admitClaim refuses a busy volume or an over-quota tenant before any VM is built. -func (m *Manager) admitClaim(tenant string, volumes []types.Volume) error { +func (m *Manager) admitClaim(tenant string, volumes []resolvedVolume) error { m.mu.Lock() defer m.mu.Unlock() if err := m.quotaErr(1, tenant); err != nil { return err } - return m.reserveVolumes(volumes) + return m.reserveResolved(volumeAccessFor(tenant, tenant), volumes, nil) } // quotaErr answers ErrQuota when the node takes no more claims; callers hold m.mu. @@ -696,10 +696,8 @@ func (m *Manager) pauseExpired(ctx context.Context, sb *types.Sandbox) error { func (m *Manager) purgeArchiveCk(ctx context.Context, id, ck, tenant string) { logger := log.WithFunc("pool.purgeArchiveCk") - if err := m.deleteCkLocked(ctx, ck); err != nil { + if _, err := m.deleteArchiveCk(ctx, ck); err != nil { logger.Warnf(ctx, "delete archive ck %s: %v", ck, err) - } else if err := m.clearArchiveCk(ck); err != nil { - logger.Warnf(ctx, "clear archive ck %s: %v", ck, err) } m.counters.archiveDeletes.Add(1) m.recordUsage(ctx, usageEvent{Event: "archive_delete", ID: id, Reference: ck, Tenant: tenant}) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 5d7111df..144a0a42 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -68,10 +68,19 @@ type claimDTO struct { // claimRow keeps its encoded member until set replaces the row; set never edits one. type claimRow struct { - dto claimDTO - raw []byte + id string + refs claimRefs + dto claimDTO + raw []byte } +type claimRefs struct { + volumes []types.Volume + checkpoints []string +} + +func (r claimRefs) empty() bool { return len(r.volumes) == 0 && len(r.checkpoints) == 0 } + // claimStore persists claimed sandboxes across daemon restarts; warm VMs are not persisted. type claimStore struct { path string @@ -81,10 +90,13 @@ type claimStore struct { writeMu sync.Mutex buf []byte - mu sync.Mutex - rows map[string]*claimRow - seq uint64 - written uint64 + mu sync.Mutex + rows map[string]*claimRow + seq uint64 + written uint64 + ackRefs map[string]claimRefs + attemptRefs map[string]claimRefs + replaceFile func([]byte) error recommitting atomic.Bool } @@ -106,10 +118,29 @@ func (s *claimStore) load() (map[string]*types.Sandbox, error) { return nil, fmt.Errorf("parse claims: %w", err) } for _, sb := range claims { + for _, volumes := range [][]types.Volume{sb.Volumes, sb.VolumesWant} { + for i := range volumes { + if volumes[i].AttachOnly && volumes[i].Mount != "" { + return nil, fmt.Errorf("claim %q has a contradictory volume mount", sb.ID) + } + volumes[i].AttachOnly = volumes[i].Mount == "" + } + } if sb.VolumesWant == nil { sb.VolumesWant = sb.Volumes } } + s.mu.Lock() + s.ackRefs = nil + for _, sb := range claims { + if row := newClaimRow(sb); !row.refs.empty() { + if s.ackRefs == nil { + s.ackRefs = map[string]claimRefs{} + } + s.ackRefs[row.id] = row.refs + } + } + s.mu.Unlock() return claims, nil } @@ -118,7 +149,7 @@ func (s *claimStore) set(sbs ...*types.Sandbox) claimSnapshot { s.mu.Lock() defer s.mu.Unlock() for _, sb := range sbs { - s.rows[sb.ID] = &claimRow{dto: dtoOf(sb)} + s.rows[sb.ID] = newClaimRow(sb) } s.seq++ return claimSnapshot{seq: s.seq} @@ -154,7 +185,7 @@ func (s *claimStore) reset(claims map[string]*types.Sandbox) claimSnapshot { defer s.mu.Unlock() s.rows = make(map[string]*claimRow, len(claims)) for id, sb := range claims { - s.rows[id] = &claimRow{dto: dtoOf(sb)} + s.rows[id] = newClaimRow(sb) } s.seq++ return claimSnapshot{seq: s.seq} @@ -170,6 +201,16 @@ func (s *claimStore) commit(snap claimSnapshot) error { return nil } rows, seq := slices.AppendSeq(make([]*claimRow, 0, len(s.rows)), maps.Values(s.rows)), s.seq + var refs map[string]claimRefs + for _, row := range rows { + if !row.refs.empty() { + if refs == nil { + refs = map[string]claimRefs{} + } + refs[row.id] = row.refs + } + } + s.attemptRefs = mergeRecoveryRefs(s.attemptRefs, refs) s.mu.Unlock() s.buf = append(s.buf[:0], '{') for i, row := range rows { @@ -186,10 +227,15 @@ func (s *claimStore) commit(snap claimSnapshot) error { s.buf = append(s.buf, row.raw...) } s.buf = append(s.buf, '}') - if err := s.replace(s.buf); err != nil { + replace := s.replace + if s.replaceFile != nil { + replace = s.replaceFile + } + if err := replace(s.buf); err != nil { return err } s.mu.Lock() + s.ackRefs, s.attemptRefs = refs, nil s.written = seq s.mu.Unlock() return nil @@ -279,3 +325,105 @@ func volumesWantDTO(sb *types.Sandbox) []types.Volume { } return append([]types.Volume{}, sb.VolumesWant...) } + +func newClaimRow(sb *types.Sandbox) *claimRow { + refs := slices.Clone(sb.Volumes) + for _, v := range sb.VolumesWant { + if !slices.Contains(refs, v) { + refs = append(refs, v) + } + } + return &claimRow{id: sb.ID, refs: claimRefs{volumes: refs, checkpoints: checkpointIDs(sb)}, dto: dtoOf(sb)} +} + +func checkpointIDs(sb *types.Sandbox) []string { + var checkpoints []string + for _, id := range []string{sb.ArchiveCk, sb.ArchivePending, sb.ReplicaCk, sb.ReplicaPending} { + if id != "" && !slices.Contains(checkpoints, id) { + checkpoints = append(checkpoints, id) + } + } + return checkpoints +} + +func (s *claimStore) recoveryVolumes(owner ...string) map[string][]types.Volume { + s.mu.Lock() + defer s.mu.Unlock() + if len(owner) > 0 { + id := owner[0] + refs := appendRecoveryRefs(slices.Clone(s.ackRefs[id].volumes), s.attemptRefs[id].volumes) + if len(refs) == 0 { + return nil + } + return map[string][]types.Volume{id: refs} + } + var volumes map[string][]types.Volume + for id, refs := range mergeRecoveryRefs(mergeRecoveryRefs(nil, s.ackRefs), s.attemptRefs) { + if len(refs.volumes) > 0 { + if volumes == nil { + volumes = map[string][]types.Volume{} + } + volumes[id] = refs.volumes + } + } + return volumes +} + +func (s *claimStore) checkpointRefs(owner ...string) map[string]struct{} { + s.mu.Lock() + defer s.mu.Unlock() + var refs map[string]struct{} + add := func(row claimRefs) { + for _, ck := range row.checkpoints { + if refs == nil { + refs = map[string]struct{}{} + } + refs[ck] = struct{}{} + } + } + if len(owner) > 0 { + id := owner[0] + if row := s.rows[id]; row != nil { + add(row.refs) + } + add(s.ackRefs[id]) + add(s.attemptRefs[id]) + return refs + } + for _, row := range s.rows { + add(row.refs) + } + for _, row := range s.ackRefs { + add(row) + } + for _, row := range s.attemptRefs { + add(row) + } + return refs +} + +func mergeRecoveryRefs(dst, src map[string]claimRefs) map[string]claimRefs { + if dst == nil && len(src) > 0 { + dst = make(map[string]claimRefs, len(src)) + } + for id, refs := range src { + row := dst[id] + row.volumes = appendRecoveryRefs(row.volumes, refs.volumes) + for _, ck := range refs.checkpoints { + if !slices.Contains(row.checkpoints, ck) { + row.checkpoints = append(row.checkpoints, ck) + } + } + dst[id] = row + } + return dst +} + +func appendRecoveryRefs(dst, src []types.Volume) []types.Volume { + for _, v := range src { + if !slices.Contains(dst, v) { + dst = append(dst, v) + } + } + return dst +} diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index 0539a537..8ddf42d5 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -1,6 +1,7 @@ package pool import ( + "encoding/json/v2" "errors" "io" "io/fs" @@ -18,6 +19,121 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +func TestClaimCheckpointReferencesTrackPossibleSnapshots(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", ArchiveCk: "archive", ArchivePending: "archive-pending", ReplicaCk: "replica", ReplicaPending: "replica-pending", FromCheckpoint: "branch-origin"} + snap := s.set(sb) + if refs := s.checkpointRefs(sb.ID); len(refs) != 4 { + t.Fatalf("current references: %+v", refs) + } + if err := s.commit(snap); err != nil { + t.Fatal(err) + } + sb.ReplicaPending = "uncertain-replica" + s.replaceFile = func(raw []byte) error { + if err := s.replace(raw); err != nil { + return err + } + return errors.New("injected lost replacement acknowledgement") + } + if err := s.commit(s.set(sb)); err == nil { + t.Fatal("replacement unexpectedly acknowledged") + } + loaded, err := newClaimStore(filepath.Dir(s.path), false).load() + if err != nil || loaded[sb.ID].ReplicaPending != "uncertain-replica" { + t.Fatalf("lost acknowledgement fixture: %+v, %v", loaded, err) + } + sb.ArchiveCk, sb.ArchivePending, sb.ReplicaCk, sb.ReplicaPending = "", "", "", "" + s.set(sb) + s.set(&types.Sandbox{ID: "other", ReplicaPending: "other-replica"}) + refs := s.checkpointRefs(sb.ID) + if len(refs) != 5 { + t.Fatalf("lost snapshot references: %+v", refs) + } + delete(refs, "uncertain-replica") + if _, retained := s.checkpointRefs(sb.ID)["uncertain-replica"]; !retained { + t.Fatal("caller mutated the retained reference set") + } + if len(s.checkpointRefs("unknown")) != 0 || len(s.checkpointRefs("other")) != 1 { + t.Fatal("checkpoint references crossed claim ownership") + } + s.replaceFile = nil + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if len(s.checkpointRefs(sb.ID)) != 0 || len(s.checkpointRefs()) != 1 { + t.Fatal("acknowledged snapshot did not retire only the removed references") + } +} + +func TestClaimCapturedCheckpointReferenceSurvivesConcurrentClear(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", ReplicaPending: "captured-replica"} + entered, resume := make(chan struct{}), make(chan struct{}) + release := sync.OnceFunc(func() { close(resume) }) + defer release() + s.replaceFile = func(raw []byte) error { + close(entered) + <-resume + var rows map[string]*types.Sandbox + if err := json.Unmarshal(raw, &rows); err != nil { + return err + } + if rows[sb.ID].ReplicaPending != "captured-replica" { + return errors.New("captured write lost its original checkpoint") + } + return s.replace(raw) + } + snap := s.set(sb) + committed := make(chan error, 1) + go func() { committed <- s.commit(snap) }() + <-entered + sb.ReplicaPending = "" + s.set(sb) + if _, retained := s.checkpointRefs(sb.ID)["captured-replica"]; !retained { + t.Error("captured replacement reference disappeared before acknowledgement") + } + release() + if err := <-committed; err != nil { + t.Fatal(err) + } + s.replaceFile = nil + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if len(s.checkpointRefs(sb.ID)) != 0 { + t.Fatal("later acknowledgement retained an obsolete checkpoint") + } +} + +func TestClaimCheckpointReferencesSurviveFailedStartupRewrite(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", ReplicaPending: "before-recovery"} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + s = newClaimStore(filepath.Dir(s.path), false) + claims, err := s.load() + if err != nil { + t.Fatal(err) + } + claims[sb.ID].ReplicaPending = "" + s.replaceFile = func([]byte) error { return errors.New("startup rewrite failed") } + if err = s.save(claims); err == nil { + t.Fatal("startup rewrite unexpectedly acknowledged") + } + if _, retained := s.checkpointRefs(sb.ID)["before-recovery"]; !retained { + t.Fatal("failed startup rewrite retired the loaded checkpoint") + } + s.replaceFile = nil + if err = s.save(claims); err != nil { + t.Fatal(err) + } + if len(s.checkpointRefs(sb.ID)) != 0 { + t.Fatal("successful startup rewrite retained the old checkpoint") + } +} + func TestRecommitKeepsOneRetrierWhileWritesFail(t *testing.T) { m := newTestManager(t, newFakeEngine()) sb := mustClaim(t, m, testKey) @@ -231,3 +347,32 @@ func TestClaimStoreConcurrentCommits(t *testing.T) { t.Fatalf("after concurrent commits: %+v, %v", got, err) } } + +func TestClaimLoadRestoresAttachOnlyBeforeRecoveryReferences(t *testing.T) { + dir := t.TempDir() + raw := `{"old":{"id":"old","volumes":[{"name":"raw","mode":"rw"},{"name":"mounted","mount":"/data"}],"volumes_want":[]},"desired":{"id":"desired","volumes":[],"volumes_want":[{"name":"raw"}]},"missing":{"id":"missing","volumes":[{"name":"raw"}]}}` + if err := os.WriteFile(filepath.Join(dir, "claims.json"), []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + s := newClaimStore(dir, false) + claims, err := s.load() + if err != nil { + t.Fatal(err) + } + if !claims["old"].Volumes[0].AttachOnly || claims["old"].Volumes[1].AttachOnly || claims["old"].VolumesWant == nil || len(claims["old"].VolumesWant) != 0 { + t.Fatalf("old shapes changed: %+v", claims["old"]) + } + if !claims["desired"].VolumesWant[0].AttachOnly || !slices.Equal(claims["missing"].Volumes, claims["missing"].VolumesWant) { + t.Fatal("legacy desired or missing-desired fallback changed") + } + refs := s.recoveryVolumes() + if !refs["old"][0].AttachOnly || !refs["desired"][0].AttachOnly { + t.Fatalf("recovery references captured before canonicalization: %+v", refs) + } + if err = os.WriteFile(filepath.Join(dir, "claims.json"), []byte(`{"bad":{"id":"bad","volumes":[{"name":"data","mount":"/data","attach_only":true}]}}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err = newClaimStore(dir, false).load(); err == nil { + t.Fatal("accepted contradictory persisted shape") + } +} diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index dd0896f4..7246e222 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -130,6 +130,10 @@ func (m *Manager) putRow(ctx context.Context, sb *types.Sandbox, ck string, repl return 0, nil } m.mu.Lock() + if replica && m.hasVolumeHolds(sb) { + m.mu.Unlock() + return 0, ErrVolumeCapture + } dto, deadline, held := dtoOf(sb), sb.Deadline, sb.ArchiveEpoch switch { case replica: @@ -299,7 +303,9 @@ func (m *Manager) reputArchive(ctx context.Context, sb *types.Sandbox, held int6 epoch, err = m.putReplica(ctx, sb, sb.ReplicaCk) } if err != nil { - log.WithFunc("pool.reputArchive").Warnf(ctx, "re-put archive row %s: %v", sb.ID, err) + if !errors.Is(err, ErrVolumeCapture) { + log.WithFunc("pool.reputArchive").Warnf(ctx, "re-put archive row %s: %v", sb.ID, err) + } return } _, _ = m.commitIfLive(ctx, sb, func() { sb.ArchiveEpoch = epoch }) @@ -477,10 +483,14 @@ func (m *Manager) dropStaleRow(ctx context.Context, sb *types.Sandbox, rec archi func (m *Manager) purgeArchiveRow(ctx context.Context, rec archives.Record) { logger := log.WithFunc("pool.purgeArchiveRow") - if err := m.deleteArchiveCk(ctx, rec.Checkpoint); err != nil { + deleted, err := m.deleteArchiveCk(ctx, rec.Checkpoint) + if err != nil { logger.Warnf(ctx, "purge archive %s: delete ck %s: %v", rec.ID, rec.Checkpoint, err) return } + if !deleted { + return + } switch err := m.archives.Delete(ctx, rec.ID, rec.Epoch); { case errors.Is(err, archives.ErrLost): return @@ -647,5 +657,5 @@ func (m *Manager) archiveRowed(sb *types.Sandbox) bool { } func (m *Manager) expiryDropsRow(sb *types.Sandbox) bool { - return m.archiveRowed(sb) || sb.HasReplica() && !m.archivesAtDeadline(sb) + return m.archiveRowed(sb) || m.archives != nil && sb.HasReplica() && !m.archivesAtDeadline(sb) } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 9e0124cb..883451f0 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -102,6 +102,11 @@ var ( ErrNoEgressFork = errors.New("egress-lane sandboxes cannot fork, checkpoint, or promote: a resumed guest egresses before its fresh tap can be locked") ErrVolumeCapture = errors.New("sandboxes with volumes cannot hibernate, fork, checkpoint, or promote") ErrVolumeBusy = errors.New("volume is held by another claim") + + ErrReplicaPresent = errors.New("live writable attachment requires replica and recovery checkpoint retirement") + ErrReplicatingPool = errors.New("live writable volumes require a non-replicating pool") + ErrReplicaStranded = errors.New("live writable attachment is blocked by retained replica or archive state with archive_failover disabled") + // Replaying a journal takes a writable mount, so readers stay out. ErrVolumeNeedsRecovery = errors.New("volume needs recovery by a writable claim") ErrArchived = errors.New("sandbox is archived; an exec or file call wakes it first") @@ -118,6 +123,13 @@ var ( errNoEgressTap = errors.New("egress-lane claim has no lockable tap") ) +var ( + ErrVolumeRegistrationConflict = errors.New("volume registration conflicts with the catalog") + ErrVolumeCatalogUnavailable = errors.New("volume catalog requires a successful persistence retry") + ErrVolumeCatalogReadOnly = errors.New("runtime volume registration requires sandboxd to run as OS root") + ErrVolumeCatalogInvalid = errors.New("runtime volume catalog storage configuration is invalid") +) + // Engine is the slice of the cocoon driver the manager consumes. type Engine interface { Clone(ctx context.Context, fromDir, name string, key types.PoolKey) (types.VMRecord, error) @@ -145,7 +157,7 @@ type Engine interface { TrimCow(ctx context.Context, vsockSocket string) error CheckLiveVolumeSupport(ctx context.Context) error DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error - DetachVolume(ctx context.Context, vmName, sock, name, mount string) error + DetachVolume(ctx context.Context, vmName, sock, name, mount string) (engine.VolumeDetachResult, error) MountVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error MountLiveVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error UnmountVolume(ctx context.Context, vsockSocket, mount string) error @@ -321,16 +333,14 @@ type PeerDeleteFunc func(ctx context.Context, id string) // Manager owns the node's pools, claims, and their persistence. type Manager struct { - eng Engine - dataDir string - egress bool - maxFork int - store *claimStore - volumes map[string]catalogVolume - // volumeAdmission counts the live holders of each volume name; guarded by m.mu. - volumeAdmission map[string]volumeHolders - volumeRuns map[*types.Sandbox]*volumeRun - volumeErrors map[*types.Sandbox]map[string]string + eng Engine + dataDir string + egress bool + maxFork int + store *claimStore + catalog volumeCatalog + volumeRuns map[*types.Sandbox]*volumeRun + volumeErrors map[*types.Sandbox]map[string]string poolStore *poolStore // poolShared is the cell's pool set with a meta store; poolVersion, under m.mu, is its newest version applied. @@ -448,8 +458,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg cocoondSocket: cfg.CocoondSocket, maxFork: cmp.Or(cfg.MaxForkCount, defaultMaxFork), store: newClaimStore(cfg.DataDir, cfg.SyncClaims), - volumes: make(map[string]catalogVolume, len(cfg.Volumes)), - volumeAdmission: map[string]volumeHolders{}, + catalog: volumeCatalog{entries: make(map[string]*catalogVolume, len(cfg.Volumes)), holders: map[string]volumeHolders{}}, volumeRuns: map[*types.Sandbox]*volumeRun{}, volumeErrors: map[*types.Sandbox]map[string]string{}, poolStore: newPoolStore(cfg.DataDir), @@ -478,12 +487,17 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg healSem: make(chan struct{}, maxConcurrentHeals), } for _, volume := range cfg.Volumes { - m.volumes[volume.Name] = catalogVolume{ - disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: volume.DirectIO}, - tenants: slices.Clone(volume.Tenants), - writable: volume.Writable, + m.catalog.generation++ + m.catalog.entries[volume.Name] = &catalogVolume{ + generation: m.catalog.generation, + disk: engine.VolumeSpec{Name: volume.Name, Path: volume.Path, DirectIO: cmp.Or(volume.DirectIO, types.DirectIOOff)}, + acl: newVolumeACL(volume.Tenants, volumePublic), + writable: volume.Writable, } } + if err := m.loadRuntimeVolumes(); err != nil { + return nil, err + } if err := os.MkdirAll(m.goldensDir(), 0o750); err != nil { return nil, fmt.Errorf("create goldens dir: %w", err) } @@ -600,6 +614,7 @@ func (m *Manager) Run(ctx context.Context) { case <-m.refillKick: m.refillOnce(ctx) case <-reap.C: + go m.retryVolumeCatalog(ctx) m.retryRemovals(ctx) m.reapOnce(ctx) m.idleOnce(ctx) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 136cd118..585f2ed6 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1658,24 +1658,28 @@ func (f *fakeEngine) MountLiveVolume(ctx context.Context, sock, name, mount stri return f.MountVolume(ctx, sock, name, mount, rw) } -func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) error { +func (f *fakeEngine) DetachVolume(ctx context.Context, _, sock, name, mount string) (engine.VolumeDetachResult, error) { if f.liveVolumeHook != nil { if err := f.liveVolumeHook("detach"); err != nil { - return err + return engine.VolumeDetachResult{}, err } } if err := ctx.Err(); err != nil { - return err + return engine.VolumeDetachResult{}, err } if mount != "" { if err := f.UnmountVolume(ctx, sock, mount); err != nil { - return err + return engine.VolumeDetachResult{}, err } } f.mu.Lock() defer f.mu.Unlock() f.volumeOps = append(f.volumeOps, "detach:"+name) - return f.detachVolumeErr + if f.detachVolumeErr != nil { + return engine.VolumeDetachResult{}, f.detachVolumeErr + } + normal := slices.ContainsFunc(f.volumeMounts, func(v types.Volume) bool { return v.Name == name && v.Mount == mount && v.RW() }) + return engine.VolumeDetachResult{NormalUnmount: normal}, nil } func (f *fakeEngine) UnmountVolume(_ context.Context, _, mount string) error { diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 3bfa3cf1..e4992bca 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -70,13 +70,14 @@ func (m *Manager) Reconcile(ctx context.Context) error { owned[sb.VMName] = true referenced[sb.HibernateSnap] = true } - saveErr := m.store.save(m.claimed) + js := m.store.reset(m.claimed) pooled := make(map[string]bool, len(m.pools)) for _, p := range m.pools { m.adoptGolden(p, imageIDs[p.key.Template]) pooled[p.hash] = true } m.mu.Unlock() + saveErr := m.store.commit(js) for _, sb := range adopted { m.recordHibernate(ctx, sb) } @@ -156,6 +157,10 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec // removeStaleVM reclaims one unowned VM, reporting whether it is gone. func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRecord) bool { logger := log.WithFunc("pool.removeStaleVM") + m.mu.Lock() + m.catalog.unknown++ + m.mu.Unlock() + td := volumeTeardown{unknown: true} if rec.State == vmStateCreating { // a canceled ctx must not skip the busy check guarding the forced remove switch outcome, err := m.eng.ReconcileStaleCreate(context.WithoutCancel(ctx), name); { @@ -163,41 +168,52 @@ func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRe // verb missing (cocoon < v0.5.8) or failed: keep the old sweep. logger.Warnf(ctx, "reconcile stale create %s: %v; removing", name, err) case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: + m.finishVolumeTeardown(ctx, td) return true case outcome == engine.StaleCreateBusy: logger.Infof(ctx, "stale create %s has an in-flight owner; queued for retry", name) - m.queueStaleCreate(name, rec.TapDevice()) + m.queueStaleCreate(name, rec.TapDevice(), td) return false } // not-creating: the record moved on under the lock; remove normally. } if m.removeVM(ctx, name) { + m.finishVolumeTeardown(ctx, td) return true } - m.queueRemoval(name, "", rec.TapDevice(), volumeTeardown{holds: m.holdStaleVolumes(ctx, name, rec)}, time.Time{}) + td = m.resolveStaleVolumes(ctx, name, td) + m.queueRemoval(name, "", rec.TapDevice(), td, time.Time{}) return false } -func (m *Manager) holdStaleVolumes(ctx context.Context, name string, rec types.VMRecord) []types.Volume { - if rec.State != vmStateRunning { - return nil +func (m *Manager) resolveStaleVolumes(ctx context.Context, name string, td volumeTeardown) volumeTeardown { + if !td.unknown { + return td } - inspected, _, err := m.eng.Inspect(ctx, name) + inspected, present, err := m.eng.Inspect(ctx, name) if err != nil { - log.WithFunc("pool.holdStaleVolumes").Warnf(ctx, "inspect stale vm %s, its volumes stay unheld: %v", name, err) - return nil + log.WithFunc("pool.resolveStaleVolumes").Warnf(ctx, "inspect stale vm %s, keeping volume fence: %v", name, err) + return td } - holds := make([]types.Volume, len(inspected.AttachedDevices.Disks)) - for i, disk := range inspected.AttachedDevices.Disks { - holds[i].Name = disk.Name - if !disk.ReadOnly { - holds[i].Mode = types.VolumeModeRW + if present && (inspected.State != vmStateRunning || len(inspected.AttachedDevices.Disks) == 0) { + return td + } + var holds []types.Volume + if present { + holds = make([]types.Volume, len(inspected.AttachedDevices.Disks)) + for i, disk := range inspected.AttachedDevices.Disks { + holds[i].Name = disk.Name + if !disk.ReadOnly { + holds[i].Mode = types.VolumeModeRW + } } } m.mu.Lock() m.adoptVolumes(holds) + m.catalog.unknown-- m.mu.Unlock() - return holds + td.unknown, td.holds = false, holds + return td } // resyncEgress re-locks adopted egress claims, quarantining any it cannot lock. @@ -313,9 +329,9 @@ func (m *Manager) reclaimOrphanArchiveCks(ctx context.Context, claims map[string if row, ok := rows[ckpt.SandboxID]; ok && (row.Owner != m.nodeID || row.Checkpoint == ckpt.ID) { continue } - if err := m.deleteArchiveCk(ctx, ckpt.ID); err != nil { + if deleted, err := m.deleteArchiveCk(ctx, ckpt.ID); err != nil { logger.Warnf(ctx, "reclaim %s: %v", ckpt.ID, err) - } else { + } else if deleted { logger.Infof(ctx, "reclaimed orphan archive ck %s", ckpt.ID) } } diff --git a/sandboxd/pool/remove.go b/sandboxd/pool/remove.go index 75efec26..924e3f5d 100644 --- a/sandboxd/pool/remove.go +++ b/sandboxd/pool/remove.go @@ -45,7 +45,9 @@ func (m *Manager) confirmGone(ctx context.Context, name string) bool { // removeOrRetry reports whether the VM is confirmed gone; a survivor is queued for the reap tick. func (m *Manager) removeOrRetry(ctx context.Context, name, sandboxID string, td volumeTeardown) bool { if m.removeVM(ctx, name) { - m.finishVolumeTeardown(ctx, td) + if !m.finishVolumeTeardown(ctx, td) { + m.queueRemoval(name, sandboxID, "", td, time.Time{}) + } return true } m.queueRemoval(name, sandboxID, "", td, time.Time{}) @@ -58,9 +60,9 @@ func (m *Manager) queueRemoval(name, sandboxID, tap string, td volumeTeardown, n m.mu.Unlock() } -func (m *Manager) queueStaleCreate(name, tap string) { +func (m *Manager) queueStaleCreate(name, tap string, td volumeTeardown) { m.mu.Lock() - m.pendingRemovals[name] = pendingRemoval{tap: tap, staleCreate: true} + m.pendingRemovals[name] = pendingRemoval{tap: tap, staleCreate: true, volumes: td} m.mu.Unlock() } @@ -77,7 +79,7 @@ func (m *Manager) retryRemovals(ctx context.Context) *sync.WaitGroup { } m.mu.Unlock() names := slices.Collect(maps.Keys(batch)) - return m.runBounded(ctx, len(names), func(ctx context.Context, i int) { + return m.runBounded(context.WithoutCancel(ctx), len(names), func(ctx context.Context, i int) { m.retryRemoval(ctx, names[i], batch[names[i]]) }) } @@ -87,25 +89,29 @@ func (m *Manager) retryRemoval(ctx context.Context, name string, pending pending switch outcome, err := m.eng.ReconcileStaleCreate(ctx, name); { case err != nil: log.WithFunc("pool.retryRemoval").Warnf(ctx, "reconcile stale create %s: %v; retrying", name, err) - m.queueStaleCreate(name, pending.tap) + m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateBusy: - m.queueStaleCreate(name, pending.tap) + m.queueStaleCreate(name, pending.tap, pending.volumes) return case outcome == engine.StaleCreateCollected, outcome == engine.StaleCreateNotFound: - m.finishRemoval(ctx, pending) + m.finishRemoval(ctx, name, pending) return } } + pending.volumes = m.resolveStaleVolumes(ctx, name, pending.volumes) if !m.removeVM(ctx, name) { m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) return } - m.finishRemoval(ctx, pending) + m.finishRemoval(ctx, name, pending) } -func (m *Manager) finishRemoval(ctx context.Context, pending pendingRemoval) { - m.finishVolumeTeardown(ctx, pending.volumes) +func (m *Manager) finishRemoval(ctx context.Context, name string, pending pendingRemoval) { + if !m.finishVolumeTeardown(ctx, pending.volumes) { + m.queueRemoval(name, pending.sandboxID, pending.tap, pending.volumes, time.Time{}) + return + } if pending.sandboxID != "" { m.out.Disarm(pending.sandboxID, true) } else if pending.tap != "" { diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 4706ddd4..f6672e96 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -112,31 +112,34 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() - ok := m.claimed[sb.ID] == sb && sb.ArchiveCk == "" && sb.ArchivePending == "" var intent claimSnapshot - if ok { + var refusal error + switch { + case m.claimed[sb.ID] != sb || sb.ArchiveCk != "" || sb.ArchivePending != "": + refusal = errWokeMeanwhile + case m.hasVolumeHolds(sb): + refusal = ErrVolumeCapture + default: sb.ReplicaPending = ckID intent = m.store.set(sb) } m.mu.Unlock() - if !ok { + if refusal != nil { m.deleteOrphanArchiveCk(ctx, ckID) - return errWokeMeanwhile + return refusal } if err := m.store.commit(intent); err != nil { - m.clearReplicaIntent(ctx, sb, ckID) - m.deleteOrphanArchiveCk(ctx, ckID) - return fmt.Errorf("replicate %s: persist intent: %w", sb.ID, err) + clearErr := m.clearReplicaIntent(ctx, sb, ckID) + return fmt.Errorf("replicate %s: persist intent: %w", sb.ID, errors.Join(err, clearErr)) } epoch, err := m.replaceReplica(ctx, sb, ckID) switch { case errors.Is(err, archives.ErrLost): - m.clearReplicaIntent(ctx, sb, ckID) - m.deleteOrphanArchiveCk(ctx, ckID) + clearErr := m.clearReplicaIntent(ctx, sb, ckID) if settleErr := m.refusedReplace(ctx, sb); settleErr != nil && !errors.Is(settleErr, ErrUnknownSandbox) { return fmt.Errorf("replicate %s: settle refused write: %w", sb.ID, settleErr) } - return fmt.Errorf("replicate %s: registry: %w", sb.ID, err) + return fmt.Errorf("replicate %s: registry: %w", sb.ID, errors.Join(err, clearErr)) case err != nil: return fmt.Errorf("replicate %s: registry: %w", sb.ID, err) } @@ -184,7 +187,7 @@ func (m *Manager) settleReplicaIntent(ctx context.Context, sb *types.Sandbox) er return ErrUnknownSandbox case ok && !rec.Consumed && rec.Checkpoint == pending: var prev string - live, _ := m.commitIfLive(ctx, sb, func() { + live, commitErr := m.commitIfLive(ctx, sb, func() { prev, sb.ReplicaCk, sb.ReplicaPending, sb.ArchiveEpoch = sb.ReplicaCk, pending, "", rec.Epoch }) if !live { @@ -193,11 +196,12 @@ func (m *Manager) settleReplicaIntent(ctx context.Context, sb *types.Sandbox) er if prev != "" { m.deleteOrphanArchiveCk(ctx, prev) } + if commitErr != nil { + return fmt.Errorf("persist settled replica: %w", commitErr) + } return nil } - m.clearReplicaIntent(ctx, sb, pending) - m.deleteOrphanArchiveCk(ctx, pending) - return nil + return m.clearReplicaIntent(ctx, sb, pending) } // replicaGone reports whether the absence of sb's row means another node adopted the claim and finished with it. @@ -218,12 +222,22 @@ func (m *Manager) replicaGone(ctx context.Context, sb *types.Sandbox) (bool, err return false, err } -func (m *Manager) clearReplicaIntent(ctx context.Context, sb *types.Sandbox, ck string) { - _, _ = m.commitIfLive(ctx, sb, func() { +func (m *Manager) clearReplicaIntent(ctx context.Context, sb *types.Sandbox, ck string) error { + live, err := m.commitIfLive(ctx, sb, func() { if sb.ReplicaPending == ck { sb.ReplicaPending = "" } }) + // Queue cleanup even on failure, but the delete boundary must wait for the + // full claim snapshot so restart can still identify an unpublished intent. + m.deleteOrphanArchiveCk(ctx, ck) + if !live { + return ErrUnknownSandbox + } + if err != nil { + return fmt.Errorf("persist replica intent clear: %w", err) + } + return nil } func (m *Manager) retireReplica(ctx context.Context, sb *types.Sandbox) { @@ -247,14 +261,17 @@ func (m *Manager) dropReplica(ctx context.Context, sb *types.Sandbox) error { return err } var ck string - live, _ := m.commitIfLive(ctx, sb, func() { + live, err := m.commitIfLive(ctx, sb, func() { ck, sb.ReplicaCk = sb.ReplicaCk, "" }) if !live { return ErrUnknownSandbox } - m.dropTombstone(ctx, sb.ID, consumed) m.deleteOrphanArchiveCk(ctx, ck) + if err != nil { + return fmt.Errorf("persist replica retirement: %w", err) + } + m.dropTombstone(ctx, sb.ID, consumed) return nil } diff --git a/sandboxd/pool/replica_cleanup_test.go b/sandboxd/pool/replica_cleanup_test.go new file mode 100644 index 00000000..9b81ead2 --- /dev/null +++ b/sandboxd/pool/replica_cleanup_test.go @@ -0,0 +1,162 @@ +package pool + +import ( + "errors" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +type strandedState struct { + name string + setup func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string +} + +var strandedStates = []strandedState{ + {name: "published replica", setup: func(t *testing.T, m *Manager, _ *failoverCluster, sb *types.Sandbox) string { + replicateNow(t, m, sb) + return sb.ReplicaCk + }}, + {name: "pending replica", setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { + m.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + return sb.ReplicaPending + }}, + {name: "archive intent", setup: func(t *testing.T, m *Manager, c *failoverCluster, sb *types.Sandbox) string { + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + m.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := m.archive(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + return sb.ArchivePending + }}, +} + +func restartStranded(t *testing.T, state strandedState) (*Manager, *types.Sandbox, string) { + t.Helper() + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + ck := state.setup(t, m, c, sb) + if ck == "" { + t.Fatal("fixture left no failover checkpoint") + } + restarted := newManagerWithoutRegistry(t, m, c) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + restored, ok := restarted.byID(sb.ID) + if !ok { + t.Fatal("startup without a registry lost the claim") + } + if restored.ReplicaCk != sb.ReplicaCk || restored.ReplicaPending != sb.ReplicaPending || restored.ArchivePending != sb.ArchivePending { + t.Fatalf("startup without a registry changed failover fields: replica=%q pending=%q archive=%q", restored.ReplicaCk, restored.ReplicaPending, restored.ArchivePending) + } + if !ckExists(t, restarted, ck) || restarted.deleteMarked(ck) { + t.Fatalf("startup without a registry queued or deleted checkpoint %s", ck) + } + return restarted, restored, ck +} + +func requireStrandedWriter(t *testing.T, m *Manager, sb *types.Sandbox) { + t.Helper() + want := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + view, err := m.SetVolumes(t.Context(), sb.ID, "", want) + if !errors.Is(err, ErrReplicaStranded) || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("live writer on stranded failover state: err=%v volumes=%v want=%v", err, view.Volumes, view.VolumesWant) + } +} + +func TestFailoverDisabledKeepsStateAndRefusesLiveWriter(t *testing.T) { + for _, state := range strandedStates { + t.Run(state.name, func(t *testing.T) { + m, sb, _ := restartStranded(t, state) + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + requireStrandedWriter(t, m, sb) + }) + } +} + +func TestFailoverDisabledExitKeepsReplicaCheckpoint(t *testing.T) { + exits := []struct { + name string + run func(t *testing.T, m *Manager, sb *types.Sandbox) + }{ + {"release", func(t *testing.T, m *Manager, sb *types.Sandbox) { + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + }}, + {"expire", func(t *testing.T, m *Manager, sb *types.Sandbox) { + m.mu.Lock() + sb.Deadline = time.Now().Add(-time.Minute) + m.mu.Unlock() + m.reapOnce(t.Context()) + waitFor(t, func() bool { return m.Counters().Reaps == 1 }) + }}, + } + for _, state := range strandedStates { + for _, exit := range exits { + t.Run(state.name+"/"+exit.name, func(t *testing.T) { + m, sb, ck := restartStranded(t, state) + exit.run(t, m, sb) + if _, held := m.byID(sb.ID); held || !ckExists(t, m, ck) { + t.Fatalf("%s without a registry: held=%t checkpoint kept=%t", exit.name, held, ckExists(t, m, ck)) + } + }) + } + } +} + +func TestReplicaFailedClearQueuesCheckpointCleanup(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + ck := sb.ReplicaCk + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected replica clear failure") } + sb.Transition.Lock() + err := m.dropReplica(t.Context(), sb) + sb.Transition.Unlock() + if err == nil || sb.HasReplica() { + t.Fatalf("expected an unacknowledged memory clear: replica=%t, %v", sb.HasReplica(), err) + } + if !m.deleteMarked(ck) { + t.Error("failed replica clear did not queue its checkpoint for cleanup") + } + m.retryArchiveDeletes(t.Context()) + if !ckExists(t, m, ck) { + t.Fatal("queued cleanup deleted a checkpoint still referenced by claims.json") + } + m.store.replaceFile = nil + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + m.retryArchiveDeletes(t.Context()) + if ckExists(t, m, ck) || m.deleteMarked(ck) { + t.Fatal("checkpoint cleanup did not finish after its reference retired") + } +} + +func newManagerWithoutRegistry(t *testing.T, old *Manager, c *failoverCluster) *Manager { + t.Helper() + cfg := &config.Config{ + APIToken: "root", DataDir: old.dataDir, CheckpointDir: c.ckDir, + Pools: []config.PoolSpec{{PoolKey: testKey}}, Volumes: old.cfg.Volumes, + } + m, err := NewManager(t.Context(), cfg, old.eng, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + if m.archives != nil { + t.Fatal("failover-disabled fixture unexpectedly has a registry") + } + return m +} diff --git a/sandboxd/pool/volume.go b/sandboxd/pool/volume.go index 5a1e8eed..e34d2343 100644 --- a/sandboxd/pool/volume.go +++ b/sandboxd/pool/volume.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "maps" "os" "path/filepath" "slices" @@ -31,30 +32,78 @@ type volumeHolders struct { } type resolvedVolume struct { - disk engine.VolumeSpec - applied types.Volume + generation uint64 + disk engine.VolumeSpec + applied types.Volume } // volumeTeardown captures marker paths at quiesce time; a later config can re-point the entry. type volumeTeardown struct { - holds []types.Volume - clears []string + done <-chan struct{} + unknown bool + holds []types.Volume + clears []string } type catalogVolume struct { - disk engine.VolumeSpec - tenants []string - writable bool + generation uint64 + runtime bool + disk engine.VolumeSpec + acl volumeACL + writable bool +} + +type volumeACLMode uint8 + +const ( + volumeOperatorOnly volumeACLMode = iota + 1 + volumePublic + volumeTenantList +) + +type volumeACL struct { + mode volumeACLMode + tenants []string +} + +type volumeAccess struct { + owner string + operator bool +} + +func newVolumeACL(tenants []string, fallback volumeACLMode) volumeACL { + if len(tenants) > 0 { + return volumeACL{mode: volumeTenantList, tenants: slices.Clone(tenants)} + } + return volumeACL{mode: fallback} +} + +func volumeAccessFor(caller, owner string) volumeAccess { + return volumeAccess{owner: owner, operator: caller == ""} } func (v catalogVolume) allowed(tenant string) bool { - return tenant == "" || len(v.tenants) == 0 || slices.Contains(v.tenants, tenant) + return v.acl.allowedFor(volumeAccessFor(tenant, tenant)) +} + +func (acl volumeACL) allowedFor(access volumeAccess) bool { + switch acl.mode { + case volumePublic: + return true + case volumeTenantList: + return access.owner == "" || slices.Contains(acl.tenants, access.owner) + case volumeOperatorOnly: + return access.operator + default: + return false + } } // Volumes reports the caller-visible fleet catalog; an empty tenant means root. func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeInfo { - infos := make([]types.VolumeInfo, 0, len(m.volumes)) - for name, volume := range m.volumes { + catalog := m.catalogSnapshot() + infos := make([]types.VolumeInfo, 0, len(catalog)) + for name, volume := range catalog { if !volume.allowed(tenant) { continue } @@ -64,7 +113,7 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI Nodes: holders[name], Writable: volume.writable, } - if st, err := os.Stat(volume.disk.Path); err == nil { + if st, err := m.statVolume(volume.disk.Path); err == nil { info.SizeBytes = st.Size() info.Available = true info.Nodes = max(info.Nodes, 1) @@ -74,7 +123,7 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI // root can see peer-only names; a tenant needs local catalog metadata to apply the access list if tenant == "" { for name, nodes := range holders { - if _, ok := m.volumes[name]; !ok { + if _, ok := catalog[name]; !ok { infos = append(infos, types.VolumeInfo{ Name: name, DefaultMount: types.DefaultVolumeMount(name), @@ -89,9 +138,10 @@ func (m *Manager) Volumes(tenant string, holders map[string]int) []types.VolumeI // VolumeNames returns the sorted names this node can currently serve. func (m *Manager) VolumeNames() []string { - names := make([]string, 0, len(m.volumes)) - for name, volume := range m.volumes { - if _, err := os.Stat(volume.disk.Path); err == nil { + catalog := m.catalogSnapshot() + names := make([]string, 0, len(catalog)) + for name, volume := range catalog { + if _, err := m.statVolume(volume.disk.Path); err == nil { names = append(names, name) } } @@ -108,8 +158,9 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri return true, nil } local := true + catalog := m.catalogSnapshot() for _, name := range names { - volume, ok := m.volumes[name] + volume, ok := catalog[name] if !ok { if tenant == "" { local = false @@ -120,7 +171,7 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri if !volume.allowed(tenant) { return false, ErrVolumeUnavailable } - if _, err := os.Stat(volume.disk.Path); err != nil { + if _, err := m.statVolume(volume.disk.Path); err != nil { local = false } } @@ -128,21 +179,31 @@ func (m *Manager) VolumePlacement(key types.PoolKey, tenant string, names []stri } func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested []types.Volume) ([]resolvedVolume, error) { + return m.resolveVolumesFor(ctx, volumeAccessFor(tenant, tenant), requested) +} + +func (m *Manager) resolveVolumesFor(ctx context.Context, access volumeAccess, requested []types.Volume) ([]resolvedVolume, error) { if len(requested) == 0 { return nil, nil } - // an attach-only claim has no mounts to default; re-defaulting here would mount them - applied, err := types.ValidateVolumes(requested, types.VolumesAttachOnly(requested)) + applied, err := types.ValidateVolumes(requested) if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } + m.mu.Lock() + if err := m.catalog.admissionError(); err != nil { + m.mu.Unlock() + return nil, err + } + catalog := maps.Clone(m.catalog.entries) + m.mu.Unlock() resolved := make([]resolvedVolume, 0, len(applied)) for _, volume := range applied { - entry, ok := m.volumes[volume.Name] - if !ok || !entry.allowed(tenant) { + entry, ok := catalog[volume.Name] + if !ok || !entry.acl.allowedFor(access) { return nil, ErrVolumeUnavailable } - resolvedEntry, resolveErr := m.resolveCatalogVolume(ctx, volume) + resolvedEntry, resolveErr := m.resolveCatalogEntry(ctx, volume, entry) if resolveErr != nil { return nil, resolveErr } @@ -152,22 +213,27 @@ func (m *Manager) resolveVolumes(ctx context.Context, tenant string, requested [ } func (m *Manager) resolveCatalogVolume(ctx context.Context, volume types.Volume) (resolvedVolume, error) { - entry, ok := m.volumes[volume.Name] - if !ok { + m.mu.Lock() + entry := m.catalog.entries[volume.Name] + m.mu.Unlock() + return m.resolveCatalogEntry(ctx, volume, entry) +} + +func (m *Manager) resolveCatalogEntry(ctx context.Context, volume types.Volume, entry *catalogVolume) (resolvedVolume, error) { + if entry == nil { return resolvedVolume{}, ErrVolumeUnavailable } if volume.RW() && !entry.writable { return resolvedVolume{}, fmt.Errorf("%w: volume %q is not writable", ErrBadVolume, volume.Name) } - if _, statErr := os.Stat(entry.disk.Path); statErr != nil { + if _, statErr := m.statVolume(entry.disk.Path); statErr != nil { // the operator's path stays server-side; the refusal matches the unknown one log.WithFunc("pool.resolveCatalogVolume").Warnf(ctx, "stat volume %q image %s: %v", volume.Name, entry.disk.Path, statErr) return resolvedVolume{}, ErrVolumeUnavailable } disk := entry.disk disk.RW = volume.RW() - volume.AttachOnly = false // past validation the empty mount carries it - return resolvedVolume{disk: disk, applied: volume}, nil + return resolvedVolume{generation: entry.generation, disk: disk, applied: volume}, nil } // confirmVolumesClean refuses a read-only claim of a dirty image; run after holds are taken. @@ -199,7 +265,7 @@ func (m *Manager) applyVolumes(ctx context.Context, sb *types.Sandbox, volumes [ // applyVolume keeps one volume's steps strictly ordered; siblings overlap freely. func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume resolvedVolume, live bool) error { // attach-only mounts nothing, so there is no umount to verify and no marker. - attachOnly := volume.applied.Mount == "" + attachOnly := volume.applied.AttachOnly // write-ahead: the marker must be durable before any guest write can be. if volume.disk.RW && !attachOnly { if err := markVolumeDirty(volume.disk.Path); err != nil { @@ -227,7 +293,12 @@ func (m *Manager) applyVolume(ctx context.Context, sb *types.Sandbox, volume res // quiesceVolumes unmounts a claim's writable mounts in reverse; a failed unmount keeps the marker. func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeTeardown { + m.mu.Lock() td := volumeTeardown{holds: heldVolumes(sb)} + if run := m.volumeRuns[sb]; run != nil { + td.done = run.done + } + m.mu.Unlock() mounts := writableMounts(sb.Volumes) if mounts == 0 { return td @@ -246,9 +317,11 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT stuck = true continue } - if entry, ok := m.volumes[volume.Name]; ok { + m.mu.Lock() + if entry, ok := m.catalog.entries[volume.Name]; ok { td.clears = append(td.clears, entry.disk.Path) } + m.mu.Unlock() } if stuck { // the unmounts may have spent the whole budget; the fallback flush gets its own @@ -262,9 +335,13 @@ func (m *Manager) quiesceVolumes(ctx context.Context, sb *types.Sandbox) volumeT } // finishVolumeTeardown runs only once the VM is gone, when its image lock is finally released. -func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { - if len(td.holds) == 0 { - return +func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) bool { + if td.done != nil { + select { + case <-td.done: + default: + return false + } } logger := log.WithFunc("pool.finishVolumeTeardown") for _, path := range td.clears { @@ -272,13 +349,40 @@ func (m *Manager) finishVolumeTeardown(ctx context.Context, td volumeTeardown) { logger.Errorf(ctx, err, "clear dirty marker %s", path) } } - m.unreserveVolumes(td.holds) + m.mu.Lock() + m.releaseVolumes(td.holds) + if td.unknown { + m.catalog.unknown-- + } + m.mu.Unlock() + return true } // reserveVolumes admits one claim's volumes atomically; callers hold m.mu. func (m *Manager) reserveVolumes(volumes []types.Volume) error { + return m.reserveVolumesFor(volumes, nil) +} + +func (m *Manager) reserveVolumesFor(volumes []types.Volume, owner *types.Sandbox) error { + if len(volumes) == 0 { + return nil + } + if err := m.catalog.admissionError(); err != nil { + return err + } + recovery := m.store.recoveryVolumes() for _, volume := range volumes { - holders := m.volumeAdmission[volume.Name] + for id, refs := range recovery { + for _, ref := range refs { + if owner != nil && m.claimed[id] == owner && m.volumeRuns[owner] != nil && owner.ID == id && ref == volume { + continue + } + if ref.Name == volume.Name && (ref.RW() || volume.RW()) { + return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) + } + } + } + holders := m.catalog.holders[volume.Name] if holders.writers > 0 || (volume.RW() && holders.readers > 0) { return fmt.Errorf("%w: volume %q", ErrVolumeBusy, volume.Name) } @@ -290,30 +394,30 @@ func (m *Manager) reserveVolumes(volumes []types.Volume) error { // adoptVolumes counts volumes an adopted claim already holds; callers hold m.mu. func (m *Manager) adoptVolumes(volumes []types.Volume) { for _, volume := range volumes { - holders := m.volumeAdmission[volume.Name] + holders := m.catalog.holders[volume.Name] if volume.RW() { holders.writers++ } else { holders.readers++ } - m.volumeAdmission[volume.Name] = holders + m.catalog.holders[volume.Name] = holders } } // releaseVolumes drops one claim's admission holds; callers hold m.mu. func (m *Manager) releaseVolumes(volumes []types.Volume) { for _, volume := range volumes { - holders := m.volumeAdmission[volume.Name] + holders := m.catalog.holders[volume.Name] if volume.RW() { holders.writers-- } else { holders.readers-- } if holders.writers < 1 && holders.readers < 1 { - delete(m.volumeAdmission, volume.Name) + delete(m.catalog.holders, volume.Name) continue } - m.volumeAdmission[volume.Name] = holders + m.catalog.holders[volume.Name] = holders } } @@ -342,7 +446,7 @@ func writableMounts(volumes []types.Volume) int { } func needsUnmount(volume types.Volume) bool { - return volume.RW() && volume.Mount != "" + return volume.RW() && !volume.AttachOnly } func appliedVolumes(volumes []resolvedVolume) []types.Volume { @@ -389,3 +493,36 @@ func syncDir(dir string) error { } return errors.Join(d.Sync(), d.Close()) } + +func (m *Manager) catalogSnapshot() map[string]*catalogVolume { + m.mu.Lock() + defer m.mu.Unlock() + return maps.Clone(m.catalog.entries) +} + +func (m *Manager) reserveResolved(access volumeAccess, volumes []resolvedVolume, owner *types.Sandbox) error { + var held []types.Volume + if owner != nil && m.claimed[owner.ID] == owner && m.volumeRuns[owner] != nil { + held = heldVolumes(owner) + } + for _, v := range volumes { + entry := m.catalog.entries[v.applied.Name] + if entry == nil || entry.generation != v.generation { + return ErrVolumeUnavailable + } + if !slices.Contains(held, v.applied) && !entry.acl.allowedFor(access) { + return ErrVolumeUnavailable + } + if v.applied.RW() && !entry.writable { + return fmt.Errorf("%w: volume %q is not writable", ErrBadVolume, v.applied.Name) + } + } + return m.reserveVolumesFor(appliedVolumes(volumes), owner) +} + +func (m *Manager) statVolume(path string) (os.FileInfo, error) { + if m.catalog.stat != nil { + return m.catalog.stat(path) + } + return os.Stat(path) +} diff --git a/sandboxd/pool/volume_acl_test.go b/sandboxd/pool/volume_acl_test.go new file mode 100644 index 00000000..8d66ded1 --- /dev/null +++ b/sandboxd/pool/volume_acl_test.go @@ -0,0 +1,216 @@ +package pool + +import ( + "errors" + "os" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestUnsetVolumeACLRejectsAccess(t *testing.T) { + var acl volumeACL + for _, caller := range []string{"", "acme"} { + for _, owner := range []string{"", "acme"} { + if acl.allowedFor(volumeAccessFor(caller, owner)) { + t.Errorf("unset volume ACL allowed caller %q for owner %q", caller, owner) + } + } + } +} + +func TestRuntimeCatalogDefaultACL(t *testing.T) { + requireCatalogRoot(t) + for _, tenants := range [][]string{nil, {}} { + t.Run("empty", func(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m, _ := newTenantVolumeManager(t, eng, catalog) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true, Tenants: tenants} + if err := m.RegisterVolume(t.Context(), "home", spec); err != nil { + t.Fatal(err) + } + for _, current := range []*Manager{m, restartACLManager(t, eng, m.dataDir, catalog)} { + if slices.ContainsFunc(current.Volumes("acme", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { + t.Error("runtime volume with no tenant grants is visible to a tenant") + } + if !slices.ContainsFunc(current.Volumes("", nil), func(v types.VolumeInfo) bool { return v.Name == "home" }) { + t.Error("operator cannot see runtime volume") + } + if _, err := current.VolumePlacement(testKey, "acme", []string{"home"}); !errors.Is(err, ErrVolumeUnavailable) { + t.Errorf("tenant placement: %v", err) + } + if local, err := current.VolumePlacement(testKey, "", []string{"home"}); err != nil || !local { + t.Errorf("operator placement: %v %v", local, err) + } + if _, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "acme", Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW, AttachOnly: true}}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Errorf("tenant initial attachment: %v", err) + } + if _, err := current.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "data"}}); err != nil { + t.Errorf("public static volume changed: %v", err) + } + rootClaim, err := current.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "home", Mode: types.VolumeModeRW, AttachOnly: true}}}) + if err != nil { + t.Fatalf("operator initial attachment: %v", err) + } + if err := current.Release(t.Context(), rootClaim.ID, Cred{Token: rootClaim.Token}); err != nil { + t.Fatalf("operator initial claim cleanup: %v", err) + } + } + }) + } +} + +func TestRuntimeCatalogOperatorLiveAttachment(t *testing.T) { + requireCatalogRoot(t) + for _, patch := range []bool{false, true} { + t.Run(map[bool]string{false: "put", true: "patch"}[patch], func(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, nil) + if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true}); err != nil { + t.Fatal(err) + } + want := []types.Volume{{Name: "home", Mode: types.VolumeModeRW, AttachOnly: true}} + change := func(caller string, volumes []types.Volume) error { + if patch { + _, err := m.PatchVolumes(t.Context(), sb.ID, caller, types.VolumePatch{Add: volumes}) + return err + } + _, err := m.SetVolumes(t.Context(), sb.ID, caller, volumes) + return err + } + if err := change("acme", want); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("tenant fresh attachment: %v", err) + } + if err := change("", want); err != nil { + t.Fatalf("operator attachment to tenant claim: %v", err) + } + if err := change("acme", want); err != nil { + t.Fatalf("held retry: %v", err) + } + if _, err := m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Remove: []string{"home"}}); err != nil { + t.Fatalf("tenant cleanup: %v", err) + } + if err := change("acme", want); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("tenant fresh attachment after detach: %v", err) + } + if err := change("", want); err != nil { + t.Fatalf("operator reattach: %v", err) + } + m2 := restartACLManager(t, eng, m.dataDir, nil) + if err := m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if got, err := m2.GetVolumes(sb.ID, "acme"); err != nil || len(got.Volumes) != 1 { + t.Fatalf("operator attachment lost at restart: %+v %v", got, err) + } + if _, err := m2.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{}); err != nil { + t.Fatalf("recovered cleanup: %v", err) + } + }) + } +} + +func TestRuntimeCatalogOperatorKeepsExplicitOwnerACL(t *testing.T) { + requireCatalogRoot(t) + m, sb := newTenantVolumeManager(t, newFakeEngine(), nil) + if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Tenants: []string{"other"}}); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "home"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("operator bypassed explicit claim-owner ACL: %v", err) + } +} + +func TestRuntimeCatalogRecoveryReferenceDoesNotAuthorizeAttach(t *testing.T) { + requireCatalogRoot(t) + for _, operation := range []string{"put", "patch", "reservation"} { + t.Run(operation, func(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, nil) + if err := m.RegisterVolume(t.Context(), "home", types.VolumeRegistration{Path: writeVolumeImage(t, "home.img", "private"), Writable: true}); err != nil { + t.Fatal(err) + } + want := []types.Volume{{Name: "home", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", want); err != nil { + t.Fatalf("still-held retry: %v", err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), sb.ID, "acme", nil) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detached result: %+v, %v", view, err) + } + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil || claims[sb.ID] == nil || !slices.Equal(claims[sb.ID].Volumes, want) { + t.Fatalf("old durable attachment missing: %+v, %v", claims, err) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: want}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("recovery reference lost writer exclusion: %v", err) + } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture ignored recovery reference: %v", err) + } + if err = m.UnregisterVolume(t.Context(), "home"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister ignored recovery reference: %v", err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + before := len(eng.volumeSpecs) + switch operation { + case "put": + _, err = m.SetVolumes(t.Context(), sb.ID, "acme", want) + case "patch": + _, err = m.PatchVolumes(t.Context(), sb.ID, "acme", types.VolumePatch{Add: want}) + case "reservation": + resolved, resolveErr := m.resolveVolumes(t.Context(), "", want) + if resolveErr != nil { + t.Fatal(resolveErr) + } + _, run, beginErr := m.beginVolumeRun(t.Context(), sb, "acme") + if beginErr != nil { + t.Fatal(beginErr) + } + m.mu.Lock() + err = m.reserveResolved(run.access, resolved, sb) + m.mu.Unlock() + m.finishVolumeRun(sb, run) + } + if !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("recovery-only reference authorized tenant attachment: %v", err) + } + if len(eng.volumeSpecs) != before { + t.Fatal("rejected tenant reached device attachment") + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatalf("operator reattach: %v", err) + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "acme", nil); err != nil { + t.Fatalf("tenant cleanup after operator reattach: %v", err) + } + }) + } +} + +func restartACLManager(t *testing.T, eng *fakeEngine, dataDir string, catalog []config.VolumeSpec) *Manager { + t.Helper() + cfg := &config.Config{DataDir: dataDir, APIToken: "root", Volumes: catalog} + m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatal(err) + } + return m +} diff --git a/sandboxd/pool/volume_attach_test.go b/sandboxd/pool/volume_attach_test.go index 7fc14e75..519f4aff 100644 --- a/sandboxd/pool/volume_attach_test.go +++ b/sandboxd/pool/volume_attach_test.go @@ -21,7 +21,7 @@ func TestAttachOnlyClaimAttachesWithoutMounting(t *testing.T) { if err != nil { t.Fatalf("ClaimProvision: %v", err) } - want := []types.Volume{{Name: "scratch", Mode: types.VolumeModeRW}} + want := []types.Volume{{Name: "scratch", Mode: types.VolumeModeRW, AttachOnly: true}} if !slices.Equal(sb.Volumes, want) { t.Errorf("applied volumes=%v, want %v", sb.Volumes, want) } @@ -78,11 +78,7 @@ func TestAttachOnlyClaimAttachesEveryVolumeConcurrently(t *testing.T) { if err != nil { t.Fatalf("ClaimProvision: %v", err) } - want := []types.Volume{ - {Name: "scratch", Mode: types.VolumeModeRW}, - {Name: "dataset"}, - {Name: "cache", Mode: types.VolumeModeRW}, - } + want := slices.Clone(requested) if !slices.Equal(sb.Volumes, want) { t.Errorf("volumes=%v, want the request order %v", sb.Volumes, want) } @@ -380,7 +376,7 @@ func TestQuiesceVolumesSkipsAttachOnlyEntries(t *testing.T) { sb := &types.Sandbox{ ID: "sb_mixed", VMName: "sbx-mixed", Key: testKey, VsockSocket: "/vsock/mixed", Volumes: []types.Volume{ - {Name: "scratch", Mode: types.VolumeModeRW}, + {Name: "scratch", Mode: types.VolumeModeRW, AttachOnly: true}, {Name: "mounted", Mount: "/volumes/mounted", Mode: types.VolumeModeRW}, }, } @@ -406,11 +402,11 @@ func TestWritableMountsCountsOnlyQuiescableEntries(t *testing.T) { }{ {"none", nil, 0}, {"read-only mount", []types.Volume{{Name: "a", Mount: "/a"}}, 0}, - {"attach-only writer", []types.Volume{{Name: "a", Mode: types.VolumeModeRW}}, 0}, + {"attach-only writer", []types.Volume{{Name: "a", Mode: types.VolumeModeRW, AttachOnly: true}}, 0}, {"writable mount", []types.Volume{{Name: "a", Mount: "/a", Mode: types.VolumeModeRW}}, 1}, {"mixed", []types.Volume{ {Name: "a", Mount: "/a", Mode: types.VolumeModeRW}, - {Name: "b", Mode: types.VolumeModeRW}, + {Name: "b", Mode: types.VolumeModeRW, AttachOnly: true}, {Name: "c", Mount: "/c"}, }, 1}, } { diff --git a/sandboxd/pool/volume_catalog.go b/sandboxd/pool/volume_catalog.go new file mode 100644 index 00000000..375d5b89 --- /dev/null +++ b/sandboxd/pool/volume_catalog.go @@ -0,0 +1,379 @@ +package pool + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "maps" + "math" + "os" + "path/filepath" + "slices" + "sync" + "syscall" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +const runtimeVolumesFile = "runtime-volumes.json" + +type ( + runtimeCatalog struct { + Version int `json:"version"` + Volumes map[string]types.VolumeRegistration `json:"volumes"` + } + + volumeCatalog struct { + entries map[string]*catalogVolume + generation uint64 + pending *catalogWrite + holders map[string]volumeHolders + unknown int + writeMu sync.Mutex + replace func(string, []byte) error + stat func(string) (os.FileInfo, error) + } + + catalogWrite struct { + entries map[string]*catalogVolume + generation uint64 + raw []byte + } +) + +func (c *volumeCatalog) admissionError() error { + if c.pending != nil || c.unknown > 0 { + return ErrVolumeCatalogUnavailable + } + return nil +} + +func (v catalogVolume) registration() types.VolumeRegistration { + return types.VolumeRegistration{Path: v.disk.Path, DirectIO: v.disk.DirectIO, Writable: v.writable, Tenants: v.acl.tenants} +} + +func catalogDefinitions(entries map[string]*catalogVolume) []config.VolumeSpec { + volumes := make([]config.VolumeSpec, 0, len(entries)) + for name, entry := range entries { + volumes = append(volumes, config.VolumeSpec{ + Name: name, Path: entry.disk.Path, DirectIO: entry.disk.DirectIO, + Writable: entry.writable, Tenants: entry.acl.tenants, + }) + } + return volumes +} + +func (c *volumeCatalog) adding(name string, spec types.VolumeRegistration) (*catalogWrite, error) { + if c.generation == math.MaxUint64 { + return nil, fmt.Errorf("%w: registration generations exhausted", ErrVolumeRegistrationConflict) + } + entries := maps.Clone(c.entries) + entries[name] = &catalogVolume{ + generation: c.generation + 1, + runtime: true, + disk: engine.VolumeSpec{Name: name, Path: spec.Path, DirectIO: spec.DirectIO}, + acl: newVolumeACL(spec.Tenants, volumeOperatorOnly), + writable: spec.Writable, + } + if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { + return nil, fmt.Errorf("%w: %w", ErrVolumeRegistrationConflict, err) + } + return &catalogWrite{entries: entries, generation: c.generation + 1}, nil +} + +func normalizeRegistration(name string, spec types.VolumeRegistration) (types.VolumeRegistration, error) { + if spec.DirectIO == "" { + spec.DirectIO = types.DirectIOOff + } + if err := config.ValidateVolumeCatalog([]config.VolumeSpec{{ + Name: name, Path: spec.Path, DirectIO: spec.DirectIO, Writable: spec.Writable, Tenants: spec.Tenants, + }}); err != nil { + return spec, fmt.Errorf("%w: %w", ErrBadVolume, err) + } + spec.Path = filepath.Clean(spec.Path) + spec.Tenants = slices.Clone(spec.Tenants) + slices.Sort(spec.Tenants) + spec.Tenants = slices.Compact(spec.Tenants) + return spec, nil +} + +func sameRegistration(a, b types.VolumeRegistration) bool { + return a.Path == b.Path && a.DirectIO == b.DirectIO && a.Writable == b.Writable && slices.Equal(a.Tenants, b.Tenants) +} + +func (m *Manager) loadRuntimeVolumes() error { + if _, err := os.Lstat(filepath.Join(m.dataDir, runtimeVolumesFile)); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return fmt.Errorf("inspect runtime volume catalog: %w", err) + } + if err := validateRuntimeCatalogPath(m.dataDir); err != nil { + return err + } + root, err := os.OpenRoot(m.dataDir) + if err != nil { + return fmt.Errorf("open runtime catalog directory: %w", err) + } + raw, readErr := root.ReadFile(runtimeVolumesFile) + if err = errors.Join(readErr, root.Close()); err != nil { + return fmt.Errorf("read runtime volume catalog: %w", err) + } + var record runtimeCatalog + if err = json.Unmarshal(raw, &record, json.RejectUnknownMembers(true)); err != nil { + return fmt.Errorf("parse runtime volume catalog: %w", err) + } + if record.Version != 1 || record.Volumes == nil { + return errors.New("invalid runtime volume catalog version or volumes") + } + entries := maps.Clone(m.catalog.entries) + generation := m.catalog.generation + for name, spec := range record.Volumes { + normalized, normalizeErr := normalizeRegistration(name, spec) + if normalizeErr != nil { + return fmt.Errorf("runtime volume catalog: %w", normalizeErr) + } + if m.catalog.entries[name] != nil { + return fmt.Errorf("runtime volume catalog duplicates static name %q", name) + } + if generation == math.MaxUint64 { + return errors.New("runtime volume catalog generations exhausted") + } + generation++ + entries[name] = &catalogVolume{ + generation: generation, + runtime: true, + disk: engine.VolumeSpec{Name: name, Path: normalized.Path, DirectIO: normalized.DirectIO}, + acl: newVolumeACL(normalized.Tenants, volumeOperatorOnly), + writable: normalized.Writable, + } + } + if err := config.ValidateVolumeCatalog(catalogDefinitions(entries)); err != nil { + return fmt.Errorf("runtime volume catalog: %w", err) + } + m.catalog.entries, m.catalog.generation = entries, generation + return nil +} + +// RegisterVolume durably binds a root-managed name to a backing path. +func (m *Manager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { + if os.Geteuid() != 0 { + return ErrVolumeCatalogReadOnly + } + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + return err + } + normalized, err := normalizeRegistration(name, spec) + if err != nil { + return err + } + proposal, err := m.registrationProposal(name, normalized) + if err != nil || proposal == nil { + return err + } + return m.writeCatalog(ctx, name, proposal) +} + +func (m *Manager) registrationProposal(name string, spec types.VolumeRegistration) (*catalogWrite, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.catalog.admissionError(); err != nil { + return nil, err + } + if entry := m.catalog.entries[name]; entry != nil { + if entry.runtime && sameRegistration(entry.registration(), spec) { + return nil, nil + } + return nil, fmt.Errorf("%w: volume %q already has different semantics or is static", ErrVolumeRegistrationConflict, name) + } + if m.volumeReferenced(name) { + return nil, fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) + } + return m.catalog.adding(name, spec) +} + +// UnregisterVolume retires an unreferenced runtime binding without touching its backing. +func (m *Manager) UnregisterVolume(ctx context.Context, name string) error { + if os.Geteuid() != 0 { + return ErrVolumeCatalogReadOnly + } + m.catalog.writeMu.Lock() + defer m.catalog.writeMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + return err + } + if !types.ValidVolumeName(name) { + return fmt.Errorf("%w: invalid volume name", ErrBadVolume) + } + proposal, err := m.removalProposal(name) + if err != nil || proposal == nil { + return err + } + return m.writeCatalog(ctx, name, proposal) +} + +func (m *Manager) removalProposal(name string) (*catalogWrite, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.catalog.admissionError(); err != nil { + return nil, err + } + entry := m.catalog.entries[name] + if (entry != nil && !entry.runtime) || m.volumeReferenced(name) { + return nil, fmt.Errorf("%w: volume %q is static or referenced", ErrVolumeRegistrationConflict, name) + } + if entry == nil { + return nil, nil + } + entries := maps.Clone(m.catalog.entries) + delete(entries, name) + return &catalogWrite{entries: entries, generation: m.catalog.generation}, nil +} + +func (m *Manager) volumeReferenced(name string) bool { + if holders := m.catalog.holders[name]; holders.readers > 0 || holders.writers > 0 { + return true + } + for _, run := range m.volumeRuns { + if volumeIndex(run.holds, name) >= 0 { + return true + } + } + for _, sb := range m.claimed { + if volumeIndex(heldVolumes(sb), name) >= 0 { + return true + } + } + for _, refs := range m.store.recoveryVolumes() { + if volumeIndex(refs, name) >= 0 { + return true + } + } + return false +} + +func (m *Manager) writeCatalog(ctx context.Context, name string, proposal *catalogWrite) error { + if err := validateRuntimeCatalogPath(m.dataDir); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogInvalid, err) + } + record := runtimeCatalog{Version: 1, Volumes: map[string]types.VolumeRegistration{}} + for key, entry := range proposal.entries { + if entry.runtime { + record.Volumes[key] = entry.registration() + } + } + raw, err := json.Marshal(record, json.Deterministic(true)) + if err != nil { + return fmt.Errorf("%w: %w", ErrBadVolume, err) + } + proposal.raw = raw + if err := m.acceptCatalog(ctx, name, proposal); err != nil { + return err + } + return m.persistCatalog(proposal) +} + +func (m *Manager) acceptCatalog(ctx context.Context, name string, proposal *catalogWrite) error { + m.mu.Lock() + defer m.mu.Unlock() + if err := ctx.Err(); err != nil { + return err + } + if err := m.catalog.admissionError(); err != nil { + return err + } + if m.volumeReferenced(name) { + return fmt.Errorf("%w: volume %q is referenced", ErrVolumeRegistrationConflict, name) + } + m.catalog.pending = proposal + return nil +} + +func (m *Manager) commitPendingCatalog(ctx context.Context) error { + m.mu.Lock() + proposal, unknown := m.catalog.pending, m.catalog.unknown > 0 + m.mu.Unlock() + if proposal == nil { + return nil + } + if unknown { + return ErrVolumeCatalogUnavailable + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) + } + if err := validateRuntimeCatalogPath(m.dataDir); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) + } + return m.persistCatalog(proposal) +} + +func (m *Manager) persistCatalog(proposal *catalogWrite) error { + replace := m.catalog.replace + if replace == nil { + replace = func(path string, raw []byte) error { return utils.WriteFileSync(path, raw, 0o600) } + } + if err := replace(filepath.Join(m.dataDir, runtimeVolumesFile), proposal.raw); err != nil { + return fmt.Errorf("%w: %w", ErrVolumeCatalogUnavailable, err) + } + m.mu.Lock() + m.catalog.entries, m.catalog.generation = proposal.entries, proposal.generation + m.catalog.pending = nil + m.mu.Unlock() + return nil +} + +func validateRuntimeCatalogPath(dir string) error { + if err := protectedCatalogPath(dir); err != nil { + return err + } + st, err := os.Lstat(filepath.Join(dir, runtimeVolumesFile)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if !st.Mode().IsRegular() || st.Mode().Perm() != 0o600 || st.Sys().(*syscall.Stat_t).Uid != 0 { + return errors.New("runtime volume catalog must be a root-owned regular file with mode 0600") + } + return nil +} + +func protectedCatalogPath(dir string) error { + absolute, err := filepath.Abs(dir) + if err != nil { + return err + } + for { + st, err := os.Lstat(absolute) + if err != nil { + return fmt.Errorf("inspect runtime catalog directory: %w", err) + } + if !st.IsDir() || st.Sys().(*syscall.Stat_t).Uid != 0 || st.Mode().Perm()&0o022 != 0 && st.Mode()&os.ModeSticky == 0 { + return fmt.Errorf("runtime catalog directory %q must be protected and root-owned", absolute) + } + parent := filepath.Dir(absolute) + if parent == absolute { + return nil + } + absolute = parent + } +} + +func (m *Manager) retryVolumeCatalog(ctx context.Context) { + if !m.catalog.writeMu.TryLock() { + return + } + defer m.catalog.writeMu.Unlock() + if err := m.commitPendingCatalog(ctx); err != nil { + log.WithFunc("pool.retryVolumeCatalog").Error(ctx, err, "persist runtime volume catalog") + } +} diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go new file mode 100644 index 00000000..f53dc873 --- /dev/null +++ b/sandboxd/pool/volume_catalog_test.go @@ -0,0 +1,698 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "slices" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +func TestRuntimeCatalogPersistsNormalizedSemantics(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "backing bytes stay intact") + spec := types.VolumeRegistration{Path: filepath.Dir(path) + "/./runtime", Writable: true, Tenants: []string{"beta", "acme", "acme"}} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + spec.Path = path + spec.DirectIO = types.DirectIOOff + spec.Tenants = []string{"acme", "beta"} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("normalized retry: %v", err) + } + st, statErr := os.Stat(filepath.Join(m.dataDir, "runtime-volumes.json")) + if statErr != nil || st.Mode().Perm() != 0o600 { + t.Fatalf("metadata mode: %v %v", st, statErr) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + if got := m2.VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("recovered names: %v", got) + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Tenant: "forbidden", Volumes: []types.Volume{{Name: "runtime"}}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("ACL changed at restart: %v", err) + } + resolved, err := m2.resolveVolumes(t.Context(), "acme", []types.Volume{{Name: "runtime", Mode: types.VolumeModeRW}}) + if err != nil || len(resolved) != 1 || resolved[0].disk.DirectIO != "off" || !resolved[0].disk.RW { + t.Fatalf("policy roundtrip: %+v %v", resolved, err) + } + changed := spec + changed.DirectIO = "on" + if err := m2.RegisterVolume(t.Context(), "runtime", changed); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("changed semantics: %v", err) + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatalf("absent retry: %v", err) + } + if got, err := os.ReadFile(path); err != nil || string(got) != "backing bytes stay intact" { + t.Fatalf("backing changed: %q %v", got, err) + } + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("unregister not durable: %v", got) + } +} + +func TestRuntimeCatalogInvalidDestinationDoesNotFenceStaticClaims(t *testing.T) { + requireCatalogRoot(t) + for _, kind := range []string{"symlink", "parent-owner", "parent-mode", "record-mode", "record-owner", "record-symlink"} { + t.Run(kind, func(t *testing.T) { + parent := t.TempDir() + dir := filepath.Join(parent, "state") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + static := liveCatalog(t) + m := newVolumeManagerAt(t, newFakeEngine(), dir, static) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("initial static claim: %v", err) + } + record := filepath.Join(dir, runtimeVolumesFile) + switch kind { + case "symlink": + link := filepath.Join(parent, "linked-state") + if err := os.Symlink(dir, link); err != nil { + t.Fatal(err) + } + m.dataDir = link + case "parent-owner": + if err := os.Chmod(parent, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chown(parent, 65534, 65534); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Chown(parent, 0, 0); err != nil { + t.Error(err) + } + }) + case "parent-mode": + if err := os.Chmod(parent, 0o777); err != nil { + t.Fatal(err) + } + default: + if err := os.WriteFile(record, []byte(`{"version":1,"volumes":{}}`), 0o600); err != nil { + t.Fatal(err) + } + switch kind { + case "record-mode": + if err := os.Chmod(record, 0o644); err != nil { + t.Fatal(err) + } + case "record-owner": + if err := os.Chown(record, 65534, 65534); err != nil { + t.Fatal(err) + } + case "record-symlink": + if err := os.Rename(record, record+".target"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(record+".target", record); err != nil { + t.Fatal(err) + } + } + } + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "unchanged")} + for range 2 { + err := m.RegisterVolume(t.Context(), "runtime", spec) + if !errors.Is(err, ErrVolumeCatalogInvalid) { + t.Errorf("invalid destination should be a permanent refusal: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Errorf("static claim after rejected registration: %v", err) + } + } + }) + } +} + +func TestRuntimeCatalogUsesStaticPathValidation(t *testing.T) { + requireCatalogRoot(t) + regular := writeVolumeImage(t, "backing", "unchanged") + link := filepath.Join(t.TempDir(), "backing-link") + if err := os.Symlink(regular, link); err != nil { + t.Fatal(err) + } + for _, path := range []string{regular, link, os.DevNull, t.TempDir(), filepath.Join(t.TempDir(), "missing")} { + t.Run(filepath.Base(path), func(t *testing.T) { + spec := config.VolumeSpec{Name: "runtime", Path: path, DirectIO: types.DirectIOOff, Writable: true} + if err := config.ValidateVolumeCatalog([]config.VolumeSpec{spec}); err != nil { + t.Fatalf("static validation: %v", err) + } + m := newVolumeManager(t, newFakeEngine(), nil) + registration := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "runtime", registration); err != nil { + t.Fatalf("runtime validation differs from static: %v", err) + } + m2 := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil) + if err := m2.RegisterVolume(t.Context(), "runtime", registration); err != nil { + t.Fatalf("idempotent retry after restart: %v", err) + } + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + if _, resolveErr := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(resolveErr, ErrVolumeUnavailable) { + t.Fatalf("missing backing resolved: %v", resolveErr) + } + } + if err := m2.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + }) + } + if got, err := os.ReadFile(regular); err != nil || string(got) != "unchanged" { + t.Fatalf("registration touched backing: %q %v", got, err) + } +} + +func TestRuntimeCatalogCollisionsAndRecoveryNames(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + static := liveCatalog(t) + m := newVolumeManager(t, eng, static) + other := writeVolumeImage(t, "other-runtime", "bytes") + for _, tc := range []struct{ name, path string }{{"data", other}, {"runtime", static[0].Path}, {"runtime", filepath.Dir(static[0].Path) + "/./" + filepath.Base(static[0].Path)}} { + if err := m.RegisterVolume(t.Context(), tc.name, types.VolumeRegistration{Path: tc.path}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("collision %+v: %v", tc, err) + } + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("static removal: %v", err) + } + sb := mustClaim(t, m, testKey) + sb.Volumes = []types.Volume{{Name: "unexplained", Mount: "/data"}} + if err := m.store.commit(m.store.set(sb)); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "unexplained", types.VolumeRegistration{Path: other}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("rebound unexplained recovery name: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "unexplained"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("absent referenced removal: %v", err) + } + for _, spec := range []types.VolumeRegistration{{Path: "relative"}, {Path: other, DirectIO: "invalid"}} { + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrBadVolume) { + t.Fatalf("invalid spec %+v: %v", spec, err) + } + } +} + +func TestRuntimeCatalogPostRenameFailureFencesAndConverges(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "bytes") + m.catalog.replace = func(path string, raw []byte) error { + if err := utils.WriteFileSync(path, raw, 0o600); err != nil { + return err + } + return errors.New("directory sync failed after rename") + } + spec := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("post-rename failure: %v", err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + if !slices.Equal(m2.VolumeNames(), []string{"runtime"}) { + t.Fatal("actual renamed record not recovered") + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "runtime"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("uncertain admission: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatalf("volume-free claim blocked: %v", err) + } + m.catalog.replace = nil + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("retry pending register: %v", err) + } + m.catalog.replace = func(path string, raw []byte) error { + if err := utils.WriteFileSync(path, raw, 0o600); err != nil { + return err + } + return errors.New("directory sync failed after rename") + } + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("uncertain unregister: %v", err) + } + if got := newVolumeManagerAt(t, eng, m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("unregister rename did not happen: %v", got) + } + if err := m.RegisterVolume(t.Context(), "replacement", types.VolumeRegistration{Path: path}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("pending unregister reuse: %v", err) + } + m.catalog.replace = nil + if err := m.RegisterVolume(t.Context(), "replacement", types.VolumeRegistration{Path: path}); err != nil { + t.Fatalf("retry did not converge before new request: %v", err) + } +} + +func TestResolvedCatalogBindingCannotSurviveUnregisterReuse(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), nil) + path := writeVolumeImage(t, "runtime", "original") + spec := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) + if err != nil { + t.Fatal(err) + } + if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + spec.Path = writeVolumeImage(t, "replacement", "replacement") + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("stale binding admitted: %v", err) + } + if got := volumeHoldersOf(m, "runtime"); got != (volumeHolders{}) { + t.Fatalf("stale binding leaked reservation: %+v", got) + } +} + +func TestRuntimeCatalogMissingBackingAndUnrestrictedBackingOwner(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if err := os.Chown(path, 65534, 65534); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatalf("nonroot backing owner: %v", err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + infos := m2.Volumes("", nil) + if len(infos) != 1 || infos[0].Available { + t.Fatalf("missing backing lost mapping: %+v", infos) + } + if _, err := m2.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("missing backing resolved: %v", err) + } + if err := m2.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatalf("idempotent missing mapping: %v", err) + } +} + +func requireCatalogRoot(t *testing.T) { + t.Helper() + if os.Geteuid() != 0 { + t.Skip("real runtime metadata mutation requires effective uid zero") + } +} + +func TestRuntimeRegistrationNormalization(t *testing.T) { + input := types.VolumeRegistration{Path: "/images/./data", Tenants: []string{"beta", "acme", "acme"}} + normalized, err := normalizeRegistration("data", input) + if err != nil || normalized.Path != "/images/data" || normalized.DirectIO != types.DirectIOOff || !slices.Equal(normalized.Tenants, []string{"acme", "beta"}) { + t.Fatalf("normalization: %+v %v", normalized, err) + } + normalized.Tenants[0] = "caller-copy" + if input.Tenants[1] != "acme" { + t.Fatal("normalization mutated caller input") + } + for _, tc := range []struct { + name string + spec types.VolumeRegistration + }{{"cocoon-reserved", input}, {"bad/name", input}, {"data", types.VolumeRegistration{Path: "relative"}}, {"data", types.VolumeRegistration{Path: "/data", DirectIO: "bad"}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{""}}}, {"data", types.VolumeRegistration{Path: "/data", Tenants: []string{"has space"}}}} { + if _, err := normalizeRegistration(tc.name, tc.spec); !errors.Is(err, ErrBadVolume) { + t.Fatalf("invalid registration %+v: %v", tc, err) + } + } +} + +func TestStaticOnlyCatalogDoesNotNeedRoot(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + if len(m.VolumeNames()) == 0 { + t.Fatal("static-only startup lost catalog") + } + if os.Geteuid() != 0 { + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: "/data"}); !errors.Is(err, ErrVolumeCatalogReadOnly) { + t.Fatalf("nonroot write: %v", err) + } + } +} + +func TestRuntimeCatalogRejectsInvalidMetadata(t *testing.T) { + requireCatalogRoot(t) + for _, kind := range []string{"malformed", "version", "null", "name", "relative", "directio", "duplicate-name", "duplicate-path", "static-name", "static-path", "unknown-field", "symlink", "mode", "owner", "directory"} { + t.Run(kind, func(t *testing.T) { + eng := newFakeEngine() + static := liveCatalog(t) + m := newVolumeManager(t, eng, static) + path := writeVolumeImage(t, "runtime", "bytes") + spec := fmt.Sprintf(`{"path":%q,"directio":"auto"}`, path) + raw := fmt.Sprintf(`{"version":1,"volumes":{"runtime":%s}}`, spec) + switch kind { + case "malformed": + raw = "{" + case "version": + raw = `{"version":2,"volumes":{}}` + case "null": + raw = `{"version":1,"volumes":null}` + case "name": + raw = fmt.Sprintf(`{"version":1,"volumes":{"bad/name":%s}}`, spec) + case "relative": + raw = `{"version":1,"volumes":{"runtime":{"path":"relative"}}}` + case "directio": + raw = `{"version":1,"volumes":{"runtime":{"path":"/data","directio":"bad"}}}` + case "duplicate-name": + raw = fmt.Sprintf(`{"version":1,"volumes":{"runtime":%s,"runtime":%s}}`, spec, spec) + case "duplicate-path": + raw = fmt.Sprintf(`{"version":1,"volumes":{"runtime":%s,"second":%s}}`, spec, spec) + case "static-name": + raw = fmt.Sprintf(`{"version":1,"volumes":{"data":%s}}`, spec) + case "static-path": + raw = fmt.Sprintf(`{"version":1,"volumes":{"runtime":{"path":%q}}}`, static[0].Path) + case "unknown-field": + raw = `{"version":1,"volumes":{},"unexpected":true}` + } + record := filepath.Join(m.dataDir, runtimeVolumesFile) + if err := os.WriteFile(record, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + switch kind { + case "symlink": + if err := os.Rename(record, record+".target"); err != nil { + t.Fatal(err) + } + if err := os.Symlink(record+".target", record); err != nil { + t.Fatal(err) + } + case "mode": + if err := os.Chmod(record, 0o644); err != nil { + t.Fatal(err) + } + case "owner": + if err := os.Chown(record, 65534, 65534); err != nil { + t.Fatal(err) + } + case "directory": + if err := os.Chmod(m.dataDir, 0o777); err != nil { + t.Fatal(err) + } + } + cfg := &config.Config{DataDir: m.dataDir, Volumes: static} + if _, err := NewManager(t.Context(), cfg, eng, testSecrets(t), testBackends(t, cfg)); err == nil { + t.Fatalf("accepted %s metadata", kind) + } + }) + } +} + +func TestRuntimeCatalogRetirementBlocksUnregisterAndReuse(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "unchanged") + spec := types.VolumeRegistration{Path: path, Writable: true} + if err := m.RegisterVolume(t.Context(), "data", spec); err != nil { + t.Fatal(err) + } + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("expected failed result commit") + } + eng.liveVolumeHook = nil + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("unregister unretired name: %v", err) + } + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "different", "different")}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("rebind unretired name: %v", err) + } + if err := m.RegisterVolume(t.Context(), "same-file", spec); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("alias unretired path: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("same-file writer: %v", err) + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "retirement-fenced", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture: %v", err) + } + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("hibernate before retirement: %v", err) + } + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "same-file", spec); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeCatalogIOLeavesManagerAndTransitionUnlocked(t *testing.T) { + requireCatalogRoot(t) + for _, operation := range []string{"stat", "replace"} { + t.Run(operation, func(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + sb := mustClaim(t, m, testKey) + entered, resume := make(chan struct{}), make(chan struct{}) + var once sync.Once + barrier := func() { once.Do(func() { close(entered) }); <-resume } + if operation == "stat" { + m.catalog.stat = func(path string) (os.FileInfo, error) { barrier(); return os.Stat(path) } + } else { + m.catalog.replace = func(path string, raw []byte) error { + if err := utils.WriteFileSync(path, raw, 0o600); err != nil { + return err + } + barrier() + return nil + } + } + done := make(chan error, 1) + go func() { + if operation == "stat" { + _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "data"}}) + done <- err + } else { + done <- m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}) + } + }() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("I/O did not block") + } + if !m.mu.TryLock() { + t.Fatal("I/O holds manager lock") + } + m.mu.Unlock() + if !sb.Transition.TryLock() { + t.Fatal("I/O holds transition lock") + } + sb.Transition.Unlock() + close(resume) + if err := <-done; err != nil { + t.Fatal(err) + } + }) + } +} + +func TestRuntimeCatalogCancellationBeforeAcceptanceDoesNotFenceClaims(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + m.catalog.replace = func(string, []byte) error { t.Fatal("canceled proposal attempted a write"); return nil } + err := m.RegisterVolume(ctx, "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}) + if !errors.Is(err, context.Canceled) || errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("cancellation before acceptance: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := os.Stat(filepath.Join(m.dataDir, runtimeVolumesFile)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("canceled proposal left catalog: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("canceled registration fenced static claim: %v", err) + } +} + +func TestRuntimeCatalogCancellationAfterAcceptancePreservesWriteOutcome(t *testing.T) { + requireCatalogRoot(t) + for _, uncertain := range []bool{false, true} { + t.Run(fmt.Sprint(uncertain), func(t *testing.T) { + m := newVolumeManager(t, newFakeEngine(), nil) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + m.catalog.replace = func(path string, raw []byte) error { + cancel() + if err := utils.WriteFileSync(path, raw, 0o600); err != nil { + return err + } + if uncertain { + return errors.New("directory sync result unavailable") + } + return nil + } + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + err := m.RegisterVolume(ctx, "runtime", spec) + if uncertain && !errors.Is(err, ErrVolumeCatalogUnavailable) || !uncertain && err != nil { + t.Fatalf("accepted write outcome: %v", err) + } + restarted := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil) + if !slices.Equal(restarted.VolumeNames(), []string{"runtime"}) { + t.Fatal("accepted replacement was not recoverable") + } + if uncertain { + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("uncertain replacement admitted volume: %v", err) + } + m.catalog.replace = nil + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatalf("retry canceled request: %v", err) + } + } + if _, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}); err != nil { + t.Fatalf("completed replacement unavailable: %v", err) + } + }) + } +} + +func TestRuntimeCatalogPendingWriteSurvivesLaterPreflightRefusal(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + m.catalog.replace = func(path string, raw []byte) error { + if err := utils.WriteFileSync(path, raw, 0o600); err != nil { + return err + } + return errors.New("directory sync result unavailable") + } + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + if err := m.RegisterVolume(t.Context(), "runtime", spec); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("initial uncertainty: %v", err) + } + m.catalog.replace = nil + record := filepath.Join(m.dataDir, runtimeVolumesFile) + if err := os.Chmod(record, 0o644); err != nil { + t.Fatal(err) + } + if err := m.UnregisterVolume(t.Context(), "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("later invalid metadata cleared uncertainty: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unconfirmed replacement lost fence: %v", err) + } + if err := os.Chmod(record, 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := m.UnregisterVolume(ctx, "runtime"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("canceled retry cleared uncertainty: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if !slices.Contains(m.VolumeNames(), "runtime") { + t.Fatal("retry lost original accepted registration") + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("repair did not restore static admission: %v", err) + } +} + +func TestResolvedCatalogGenerationChangesOnlyForReusedName(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), nil) + spec := types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")} + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + resolved, err := m.resolveVolumes(t.Context(), "", []types.Volume{{Name: "runtime"}}) + if err != nil { + t.Fatal(err) + } + if err = m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: writeVolumeImage(t, "other", "bytes")}); err != nil { + t.Fatal(err) + } + holds, err := m.admitVolumes("", resolved) + if err != nil { + t.Fatalf("unrelated registration invalidated resolution: %v", err) + } + m.unreserveVolumes(holds) + if err := m.UnregisterVolume(t.Context(), "runtime"); err != nil { + t.Fatal(err) + } + if err := m.RegisterVolume(t.Context(), "runtime", spec); err != nil { + t.Fatal(err) + } + if _, err := m.admitVolumes("", resolved); !errors.Is(err, ErrVolumeUnavailable) { + t.Fatalf("same-file reuse admitted stale resolution: %v", err) + } +} + +func TestRuntimeCatalogGenerationExhaustionDoesNotWrite(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), liveCatalog(t)) + m.catalog.generation = math.MaxUint64 + m.catalog.replace = func(string, []byte) error { t.Fatal("exhausted generation attempted write"); return nil } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("generation exhaustion: %v", err) + } + m.retryVolumeCatalog(t.Context()) + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); err != nil { + t.Fatalf("exhaustion fenced existing registration: %v", err) + } +} + +func TestRuntimeCatalogIgnoresAbandonedTemporaryFiles(t *testing.T) { + requireCatalogRoot(t) + m := newVolumeManager(t, newFakeEngine(), nil) + path := writeVolumeImage(t, "runtime", "bytes") + raw := fmt.Sprintf(`{"version":1,"volumes":{"abandoned":{"path":%q}}}`, path) + tmp := filepath.Join(m.dataDir, runtimeVolumesFile+".interrupted.tmp") + if err := os.WriteFile(tmp, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); len(got) != 0 { + t.Fatalf("temporary record promoted without canonical record: %v", got) + } + if err := m.RegisterVolume(t.Context(), "runtime", types.VolumeRegistration{Path: path}); err != nil { + t.Fatal(err) + } + if got := newVolumeManagerAt(t, newFakeEngine(), m.dataDir, nil).VolumeNames(); !slices.Equal(got, []string{"runtime"}) { + t.Fatalf("temporary record overrode canonical record: %v", got) + } + if got, err := os.ReadFile(tmp); err != nil || string(got) != raw { + t.Fatalf("abandoned evidence changed: %q %v", got, err) + } +} diff --git a/sandboxd/pool/volume_cleanup_test.go b/sandboxd/pool/volume_cleanup_test.go new file mode 100644 index 00000000..a14101c5 --- /dev/null +++ b/sandboxd/pool/volume_cleanup_test.go @@ -0,0 +1,485 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestIncompleteStaleInventoryRetainsVolumeProtection(t *testing.T) { + requireCatalogRoot(t) + for _, state := range []string{vmStateRunning, vmStateCreating, "stopped"} { + t.Run(state, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + path := writeVolumeImage(t, "incomplete-inventory", "original") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { + t.Fatal(err) + } + name := "sbx-incomplete-inventory" + base.vms[name] = "/vsock/incomplete" + base.creating[name] = state == vmStateCreating + base.staleOutcome = engine.StaleCreateBusy + base.removeErrFor = name + base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + cli := filepath.Join(t.TempDir(), "inspect") + writeInspect := func(state, inventory string) { + t.Helper() + raw := fmt.Sprintf("#!/bin/sh\nprintf '%%s\\n' '{\"config\":{\"name\":\"%s\"},\"state\":\"%s\"%s}'\n", name, state, inventory) + if err := os.WriteFile(cli, []byte(raw), 0o700); err != nil { + t.Fatal(err) + } + } + writeInspect(state, "") + m.eng = &incompleteInventoryEngine{fakeEngine: base, cli: engine.New(cli, nil, nil, false, false, "")} + rec := base.record(name) + rec.State = state + if m.removeStaleVM(t.Context(), name, rec) { + t.Fatal("stale VM unexpectedly disappeared") + } + for range 2 { + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unregister must refuse incomplete inventory: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("conflicting writer must refuse incomplete inventory: %v", err) + } + m.retryRemovals(t.Context()).Wait() + } + base.creating[name] = false + writeInspect(vmStateRunning, `,"attached_devices":{"disks":[{"name":"data","readonly":false}]}`) + if state == vmStateCreating { + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("busy owner must retain the unknown fence: %v", err) + } + } + base.staleOutcome = engine.StaleCreateNotCreating + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("complete inventory lost the surviving writer: %v", err) + } + other := writeVolumeImage(t, "complete-inventory-other", "other") + if err := m.RegisterVolume(t.Context(), "other", types.VolumeRegistration{Path: other}); err != nil { + t.Fatalf("complete inventory retained the global fence: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatalf("confirmed removal retained volume protection: %v", err) + } + }) + } +} + +type incompleteInventoryEngine struct { + *fakeEngine + cli *engine.Engine +} + +func (e *incompleteInventoryEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + return e.cli.Inspect(ctx, name) +} + +func TestRemovedVMKeepsVolumeUntilCanceledWorkerReturns(t *testing.T) { + requireCatalogRoot(t) + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes"), Writable: true}); err != nil { + t.Fatal(err) + } + sb := mustClaim(t, m, testKey) + eng := newGatedVolumeEngine(base) + m.eng = eng + changed := make(chan error, 1) + go func() { _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); changed <- err }() + <-eng.entered + resumed := false + defer func() { + if !resumed { + close(eng.resume) + <-changed + } + }() + released := make(chan error, 1) + go func() { released <- m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) }() + select { + case err := <-released: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("release waited for paused worker") + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { + t.Errorf("removed VM freed active worker count: %+v", got) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer overtook paused canceled worker: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Errorf("unregister overtook paused worker: %v", err) + } + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{readers: 1}) { + t.Errorf("maintenance freed active worker count: %+v", got) + } + close(eng.resume) + resumed = true + if err := <-changed; err == nil { + t.Fatal("removed claim accepted late attachment") + } + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("completed worker leaked count: %+v", got) + } + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } +} + +func TestDispatchedCleanupRetainsCatalogReference(t *testing.T) { + requireCatalogRoot(t) + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: writeVolumeImage(t, "runtime", "bytes")}); err != nil { + t.Fatal(err) + } + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if err != nil { + t.Fatal(err) + } + base.removeErrFor = sb.VMName + if err = m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("expected survivor") + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + wg := m.retryRemovals(t.Context()) + m.mu.Lock() + queued := len(m.pendingRemovals) + m.mu.Unlock() + if queued != 0 { + t.Fatal("cleanup was not dispatched") + } + err = m.UnregisterVolume(t.Context(), "data") + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + if !errors.Is(err, ErrVolumeRegistrationConflict) { + t.Fatalf("dispatched cleanup lost guard: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatal(err) + } +} + +type unknownVolumeEngine struct { + *fakeEngine + inspectFailed atomic.Bool + inspectEntered chan struct{} + inspectResume chan struct{} + inspectGated atomic.Bool +} + +func (e *unknownVolumeEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + if e.inspectEntered != nil && e.inspectGated.CompareAndSwap(false, true) { + close(e.inspectEntered) + <-e.inspectResume + } + if e.inspectFailed.Load() { + return types.VMRecord{}, false, errors.New("inspection unavailable") + } + return e.fakeEngine.Inspect(ctx, name) +} + +func TestUnknownStaleVolumesFenceUntilInspectionOrRemoval(t *testing.T) { + requireCatalogRoot(t) + for _, state := range []string{"running", "stopped", "creating"} { + t.Run(state, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true}); err != nil { + t.Fatal(err) + } + name := "sbx-stale-unknown" + base.vms[name] = "/vsock/stale" + base.stopped[name] = state == "stopped" + base.creating[name] = state == "creating" + base.staleOutcome = engine.StaleCreateBusy + base.removeErrFor = name + base.disks = map[string][]types.VMDisk{name: {{Name: "data", ReadOnly: false}}} + eng := &unknownVolumeEngine{fakeEngine: base} + eng.inspectFailed.Store(true) + m.eng = eng + if m.removeStaleVM(t.Context(), name, base.record(name)) { + t.Fatal("unknown stale VM unexpectedly gone") + } + for range 2 { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unknown stale attachment admitted reader: %v", err) + } + if err := m.UnregisterVolume(t.Context(), "data"); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("unknown stale attachment allowed unregister: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err != nil { + t.Fatalf("volume-free claim blocked: %v", err) + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + wg := m.retryRemovals(t.Context()) + if err := m.RegisterVolume(t.Context(), "new", types.VolumeRegistration{Path: writeVolumeImage(t, "new", "bytes")}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Errorf("dispatched unknown fence lost: %v", err) + } + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + } + eng.inspectFailed.Store(false) + base.stopped[name], base.creating[name] = false, false + base.staleOutcome = engine.StaleCreateNotCreating + m.retryRemovals(t.Context()).Wait() + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{writers: 1}) { + t.Fatalf("inspection did not replace fence with known holds: %+v", got) + } + if err := m.RegisterVolume(t.Context(), "unrelated", types.VolumeRegistration{Path: writeVolumeImage(t, "unrelated", "bytes")}); err != nil { + t.Fatalf("resolved inspection left global fence: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("known attachment lost exclusion: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if err := m.UnregisterVolume(t.Context(), "data"); err != nil { + t.Fatalf("confirmed removal retained fence: %v", err) + } + }) + } +} + +func TestRuntimeCatalogLoadedBeforeClaimReconciliation(t *testing.T) { + requireCatalogRoot(t) + eng := newFakeEngine() + m := newVolumeManager(t, eng, nil) + path := writeVolumeImage(t, "runtime", "bytes") + if err := m.RegisterVolume(t.Context(), "data", types.VolumeRegistration{Path: path, Writable: true, Tenants: []string{"acme"}}); err != nil { + t.Fatal(err) + } + sb, claimErr := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if claimErr != nil { + t.Fatal(claimErr) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, nil) + if got := m2.Volumes("acme", nil); len(got) != 1 || got[0].Name != "data" || !got[0].Available || !got[0].Writable { + t.Fatalf("catalog absent before reconcile: %+v", got) + } + if local, err := m2.VolumePlacement(testKey, "acme", []string{"data"}); err != nil || !local { + t.Fatalf("placement before reconcile: %v %v", local, err) + } + if err := m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + view, err := m2.GetVolumes(sb.ID, "") + if err != nil || !slices.Equal(view.Volumes, sb.Volumes) { + t.Fatalf("claim recovery: %+v %v", view, err) + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("recovered holder missing: %v", err) + } +} + +func TestUnknownFencePrecedesInspectionAndClearsAfterConfirmedRemoval(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + name := "sbx-stale-before-inspect" + base.vms[name] = "/vsock/stale" + base.removeErrFor = name + eng := &unknownVolumeEngine{fakeEngine: base, inspectEntered: make(chan struct{}), inspectResume: make(chan struct{})} + eng.inspectFailed.Store(true) + m.eng = eng + done := make(chan bool, 1) + go func() { done <- m.removeStaleVM(t.Context(), name, base.record(name)) }() + <-eng.inspectEntered + if !m.mu.TryLock() { + t.Fatal("stale inspection holds manager lock") + } + m.mu.Unlock() + _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}) + close(eng.inspectResume) + if gone := <-done; gone { + t.Fatal("failed inspect/removal unexpectedly confirmed gone") + } + if !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("admitted volume while inspection was blocked: %v", err) + } + base.removeErrFor = "" + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("successful removal retained unknown fence: %v", err) + } +} + +func TestCanceledRemovalDispatchKeepsUnknownFenceOwned(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + name := "sbx-canceled-dispatch" + base.vms[name] = "/vsock/stale" + base.removeErrFor = name + eng := &unknownVolumeEngine{fakeEngine: base} + eng.inspectFailed.Store(true) + m.eng = eng + if m.removeStaleVM(t.Context(), name, base.record(name)) { + t.Fatal("expected survivor") + } + for range cap(m.maintSem) { + m.maintSem <- struct{}{} + } + ctx, cancel := context.WithCancel(t.Context()) + wg := m.retryRemovals(ctx) + cancel() + synctest.Wait() + base.removeErrFor = "" + for range cap(m.maintSem) { + <-m.maintSem + } + wg.Wait() + m.retryRemovals(t.Context()).Wait() + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("canceled dispatch stranded unknown fence: %v", err) + } + }) +} + +func TestWarmAttachBeforeFinalizeNeedsUnknownFenceAfterRestart(t *testing.T) { + base := newFakeEngine() + name := "sbx-warm-unfinalized" + base.vms[name] = "/vsock/warm" + base.diskAttachErrFor = "broken" + base.removeErrFor = name + rendezvousAttaches(base, 2) + catalog := []config.VolumeSpec{ + {Name: "data", Path: writeVolumeImage(t, "warm-data", "data"), Writable: true}, + {Name: "broken", Path: writeVolumeImage(t, "warm-broken", "broken")}, + } + dir := t.TempDir() + m := newVolumePoolManager(t, base, dir, catalog) + m.eng = &emptyWarmInventoryEngine{fakeEngine: base} + warm := &types.Sandbox{VMName: name, Key: testKey, VsockSocket: "/vsock/warm"} + m.pools[testKey].warm = append(m.pools[testKey].warm, warm) + _, err := m.ClaimWarm(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{ + {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}, + {Name: "broken", AttachOnly: true}, + }}) + if err == nil { + t.Fatal("expected attach failure before finalization") + } + if warm.ID != "" { + t.Fatal("claim unexpectedly finalized") + } + base.mu.Lock() + attached := slices.Clone(base.disks[name]) + base.attachRendezvous = nil + base.mu.Unlock() + if !slices.ContainsFunc(attached, func(d types.VMDisk) bool { return d.Name == "data" && !d.ReadOnly }) { + t.Fatalf("warm VM has no attached writer: %+v", attached) + } + claims, err := newClaimStore(dir, false).load() + if err != nil || len(claims) != 0 { + t.Fatalf("pre-finalize journal: %+v, %v", claims, err) + } + restarted := newVolumeManagerAt(t, base, dir, catalog) + restarted.eng = &emptyWarmInventoryEngine{fakeEngine: base} + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeCatalogUnavailable) { + t.Fatalf("ambiguous warm survivor admitted competing writer: %v", err) + } + base.removeErrFor = "" + restarted.retryRemovals(t.Context()).Wait() + if _, err := restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("confirmed removal failed to clear fence: %v", err) + } +} + +type emptyWarmInventoryEngine struct{ *fakeEngine } + +func (e *emptyWarmInventoryEngine) Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) { + rec, present, err := e.fakeEngine.Inspect(ctx, name) + rec.AttachedDevices.Disks = nil + return rec, present, err +} + +func (e *emptyWarmInventoryEngine) DiskAttach(ctx context.Context, name string, spec engine.VolumeSpec) error { + if err := e.fakeEngine.DiskAttach(ctx, name, spec); err != nil { + return err + } + e.mu.Lock() + if e.disks == nil { + e.disks = map[string][]types.VMDisk{} + } + e.disks[name] = append(e.disks[name], types.VMDisk{Name: spec.Name, ReadOnly: !spec.RW}) + e.mu.Unlock() + return nil +} + +func TestArchivedReleaseWithUnvalidatedVolumeRunHasNoCleanupHolds(t *testing.T) { + base := newFakeEngine() + m := newTestManager(t, base, archivePool(3600)) + sb := mustClaim(t, m, testKey) + if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if err := m.archive(t.Context(), sb); err != nil { + t.Fatal(err) + } + if sb.VMName != "" { + t.Fatalf("archived VM name = %q", sb.VMName) + } + ctx, run, err := m.beginVolumeRun(t.Context(), sb, sb.Tenant) + if err != nil { + t.Fatal(err) + } + if err := m.checkVolumeRun(ctx, sb, run); !errors.Is(err, ErrArchived) { + t.Fatalf("archived mutation validation: %v", err) + } + td := m.quiesceVolumes(t.Context(), sb) + if td.done == nil || len(td.holds) != 0 || len(td.clears) != 0 || td.unknown { + t.Fatalf("archived teardown has cleanup obligations: %+v", td) + } + if m.finishVolumeTeardown(t.Context(), td) { + t.Fatal("open volume run unexpectedly completed") + } + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + m.finishVolumeRun(sb, run) + if len(m.pendingRemovals) != 0 { + t.Fatalf("empty teardown enqueued removal: %+v", m.pendingRemovals) + } + if slices.Contains(base.removedNames(), "") { + t.Fatal("release removed an empty VM name") + } + if len(m.store.recoveryVolumes()) != 0 { + t.Fatal("release retained archived journal references") + } +} diff --git a/sandboxd/pool/volume_convergence_test.go b/sandboxd/pool/volume_convergence_test.go index 7c002ecc..c112c859 100644 --- a/sandboxd/pool/volume_convergence_test.go +++ b/sandboxd/pool/volume_convergence_test.go @@ -22,7 +22,7 @@ func TestSettledVolumesDoNotCallDevicesOrResolveCatalog(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { t.Fatal(err) } - delete(m.volumes, "data") + delete(m.catalog.entries, "data") before := len(eng.volumeSpecs) if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}, {Name: "other"}}); err != nil { t.Fatal(err) @@ -57,9 +57,9 @@ func TestRemovedTenantRetainsAndRepairsHeldVolumes(t *testing.T) { t.Fatal("expected detaching state") } eng.detachVolumeErr = nil - entry := m.volumes["data"] - entry.tenants = []string{"other-tenant"} - m.volumes["data"] = entry + entry := m.catalog.entries["data"] + entry.acl = newVolumeACL([]string{"other-tenant"}, volumePublic) + m.catalog.entries["data"] = entry if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { t.Fatalf("held repair checked new authorization: %v", err) } @@ -81,7 +81,7 @@ func TestReaddedDetachIsPendingAtTheDurableIntentBoundary(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{}); err == nil { t.Fatal("expected detach failure") } - ctx, run, err := m.beginVolumeRun(t.Context(), sb) + ctx, run, err := m.beginVolumeRun(t.Context(), sb, sb.Tenant) if err != nil { t.Fatal(err) } @@ -115,7 +115,7 @@ func TestReleaseBeforeReservationDoesNotLeakAdmission(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) - ctx, run, err := m.beginVolumeRun(t.Context(), sb) + ctx, run, err := m.beginVolumeRun(t.Context(), sb, sb.Tenant) if err != nil { t.Fatal(err) } @@ -325,7 +325,7 @@ func TestVolumeHoldsRejectCaptureBeforeTrim(t *testing.T) { case "desired": sb.VolumesWant = held case "admitted": - _, run, err := m.beginVolumeRun(t.Context(), sb) + _, run, err := m.beginVolumeRun(t.Context(), sb, sb.Tenant) if err != nil { t.Fatal(err) } diff --git a/sandboxd/pool/volume_modes_test.go b/sandboxd/pool/volume_modes_test.go new file mode 100644 index 00000000..599ca6c0 --- /dev/null +++ b/sandboxd/pool/volume_modes_test.go @@ -0,0 +1,472 @@ +package pool + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestLiveVolumeFourModesAndMixedRoundtrip(t *testing.T) { + eng := newFakeEngine() + catalog := []config.VolumeSpec{} + request := []types.Volume{{Name: "mounted"}, {Name: "writer", Mode: types.VolumeModeRW}, {Name: "raw", AttachOnly: true}, {Name: "raw-writer", Mode: types.VolumeModeRW, AttachOnly: true}} + for _, v := range request { + catalog = append(catalog, config.VolumeSpec{Name: v.Name, Path: writeVolumeImage(t, v.Name, "seed"), Writable: true}) + } + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + view, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: request}) + if err != nil { + t.Fatal(err) + } + if len(view.Volumes) != 4 || len(eng.volumeMounts) != 2 { + t.Fatalf("mixed modes: view=%+v mounts=%+v", view, eng.volumeMounts) + } + for _, v := range view.Volumes { + if v.AttachOnly != (v.Mount == "") { + t.Fatalf("lost canonical shape: %+v", v) + } + } + raw, err := json.Marshal(view) + if err != nil { + t.Fatal(err) + } + var echoed types.SandboxVolumesResponse + if err = json.Unmarshal(raw, &echoed); err != nil { + t.Fatal(err) + } + before := slices.Clone(eng.volumeOpsLog()) + if _, err = m.SetVolumes(t.Context(), sb.ID, "", echoed.VolumesWant); err != nil { + t.Fatal(err) + } + if !slices.Equal(before, eng.volumeOpsLog()) { + t.Fatal("GET-to-PUT roundtrip changed devices") + } + for _, change := range []types.Volume{{Name: "raw", Mount: "/new"}, {Name: "raw-writer", AttachOnly: true}, {Name: "writer", Mount: "/other", Mode: types.VolumeModeRW}} { + if _, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{change}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("held shape change accepted: %+v, %v", change, err) + } + } + legacy := slices.Clone(view.VolumesWant) + for i := range legacy { + legacy[i].AttachOnly = false + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", legacy); err != nil { + t.Fatalf("legacy held omission: %v", err) + } + assertLiveJournal(t, m, sb) + restarted := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err = restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + recovered, err := restarted.GetVolumes(sb.ID, "") + if err != nil || !slices.Equal(recovered.Volumes, view.Volumes) { + t.Fatalf("mixed restart: %+v, %v", recovered, err) + } + if _, err = restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "raw-writer", AttachOnly: true}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("restart lost writer exclusion: %v", err) + } + if _, err = restarted.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + for _, spec := range catalog { + if volumeDirty(spec.Path) || volumeHoldersOf(restarted, spec.Name) != (volumeHolders{}) { + t.Fatalf("detach leaked marker or hold: %s", spec.Name) + } + } +} + +func TestLiveVolumeLegacyAttachOnlyRetryAfterFailedDetach(t *testing.T) { + for _, mode := range []string{"", types.VolumeModeRW} { + for _, method := range []string{"put", "patch"} { + t.Run(fmt.Sprintf("%s/%s", map[string]string{"": "ro", types.VolumeModeRW: "rw"}[mode], method), func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + want := []types.Volume{{Name: "data", Mode: mode, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatal(err) + } + eng.detachVolumeErr = errors.New("host detach unavailable") + view, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + if err == nil || len(view.VolumesWant) != 0 || !slices.Equal(view.Volumes, want) { + t.Fatalf("failed detach lost applied device: %+v, %v", view, err) + } + eng.detachVolumeErr = nil + before := slices.Clone(eng.volumeOpsLog()) + change := func(volumes []types.Volume) (types.SandboxVolumesResponse, error) { + if method == "patch" { + return m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: volumes}) + } + return m.SetVolumes(t.Context(), sb.ID, "", volumes) + } + if _, err = change([]types.Volume{{Name: "data", Mode: mode, Mount: "/changed"}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("held attachment changed shape: %v", err) + } + if !slices.Equal(before, eng.volumeOpsLog()) { + t.Fatal("rejected shape change operated on the device") + } + legacy := slices.Clone(want) + legacy[0].AttachOnly = false + view, err = change(legacy) + if err != nil { + t.Fatalf("legacy echo of held attach-only volume rejected: %v", err) + } + if !slices.Equal(view.VolumesWant, want) || !slices.Equal(view.Volumes, want) || view.Converging { + t.Fatalf("legacy retry did not retain the applied device: %+v", view) + } + before = slices.Clone(eng.volumeOpsLog()) + if _, err = change(legacy); err != nil { + t.Fatalf("settled legacy retry: %v", err) + } + if !slices.Equal(before, eng.volumeOpsLog()) { + t.Fatalf("settled retry changed the retained device: %v -> %v", before, eng.volumeOpsLog()) + } + }) + } + } +} + +func TestLiveVolumeLegacyAttachOnlyPatchAfterFailedCommit(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + want := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", want); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || !view.Converging { + t.Fatalf("failed detach commit did not retain a recovery reference: %+v, %v", view, err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + legacy := []types.Volume{{Name: "data", Mode: types.VolumeModeRW}} + view, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: legacy}) + if err != nil { + t.Fatalf("legacy echo of recovery-only attach-only volume rejected: %v", err) + } + if !slices.Equal(view.VolumesWant, want) || !slices.Equal(view.Volumes, want) || view.Converging { + t.Fatalf("legacy retry lost recovery attachment shape: %+v", view) + } +} + +func TestLiveWritableDetachRequiresCleanEvidence(t *testing.T) { + for _, proof := range []bool{false, true} { + t.Run(fmt.Sprint(proof), func(t *testing.T) { + base := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, base, catalog) + m.eng = &detachProofEngine{fakeEngine: base, proof: proof} + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err != nil { + t.Fatal(err) + } + path := catalog[0].Path + if !volumeDirty(path) || !base.attachDirty["data"] { + t.Fatal("writer was attached before dirty marker") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if volumeDirty(path) == proof { + t.Fatalf("marker disagrees with clean evidence: proof=%t dirty=%t", proof, volumeDirty(path)) + } + _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data"}}}) + if proof && err != nil || !proof && !errors.Is(err, ErrVolumeNeedsRecovery) { + t.Fatalf("reader admission after detach: %v", err) + } + }) + } +} + +func TestLiveWritableFailedHostDetachKeepsMarkerAndHold(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err != nil { + t.Fatal(err) + } + eng.detachVolumeErr = errors.New("host removal uncertain") + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("failed host detach reported success") + } + if !volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data").writers != 1 { + t.Fatal("failed host detach lost dirty marker or writer") + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "uncertain", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture after uncertain detach: %v", err) + } + eng.detachVolumeErr = nil + m.eng = &detachProofEngine{fakeEngine: eng} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if !volumeDirty(catalog[0].Path) { + t.Fatal("absent-device retry invented clean evidence") + } +} + +func TestLiveWritableMarkerClearFailureRemainsConservative(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err != nil { + t.Fatal(err) + } + marker := volumeDirtyPath(catalog[0].Path) + if err := os.Remove(marker); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(marker, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(marker, "block"), nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if !volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("failed marker deletion changed physical retirement or reported clean") + } +} + +type detachProofEngine struct { + *fakeEngine + proof bool +} + +func (e *detachProofEngine) DetachVolume(ctx context.Context, vm, sock, name, mount string) (engine.VolumeDetachResult, error) { + _, err := e.fakeEngine.DetachVolume(ctx, vm, sock, name, mount) + if err != nil { + return engine.VolumeDetachResult{}, err + } + return engine.VolumeDetachResult{NormalUnmount: e.proof}, nil +} + +func TestLiveWritableUncertainAdditionClearsOnlyAfterDetach(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + eng.mountVolumeErr = errors.New("mount response lost") + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err == nil { + t.Fatal("expected unknown mount result") + } + if len(sb.Volumes) != 0 || len(sb.VolumesWant) != 1 || !volumeDirty(catalog[0].Path) { + t.Fatal("pending writable attachment lost intent or marker") + } + eng.mountVolumeErr = nil + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data") != (volumeHolders{}) { + t.Fatal("uncertain addition cleanup retained marker or hold") + } +} + +func TestLiveVolumeModesKeepRecoveryAfterDetachWriteFailure(t *testing.T) { + for _, v := range []types.Volume{{Name: "data"}, {Name: "data", Mode: types.VolumeModeRW}, {Name: "data", AttachOnly: true}, {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} { + t.Run(fmt.Sprintf("rw=%t/raw=%t", v.RW(), v.AttachOnly), func(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{v}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 { + t.Fatalf("detach result failure: %+v, %v", view, err) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("writer passed unretired %v: %v", v, err) + } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture passed unretired %v: %v", v, err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + restarted := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err = restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = restarted.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err != nil { + t.Fatal(err) + } + if _, err = restarted.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("retired reference still blocks writer: %v", err) + } + }) + } +} + +func TestLiveWritableReleaseWaitsForLateDetachBeforeNewWriter(t *testing.T) { + base := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, base, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}); err != nil { + t.Fatal(err) + } + eng := &gatedDetachEngine{fakeEngine: base, entered: make(chan struct{}), resume: make(chan struct{})} + m.eng = eng + changed := make(chan error, 1) + go func() { _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); changed <- err }() + <-eng.entered + resumed := false + defer func() { + if !resumed { + close(eng.resume) + <-changed + } + }() + if !volumeDirty(catalog[0].Path) || volumeHoldersOf(m, "data").writers != 1 { + t.Fatal("host result released marker or hold before consumption") + } + if !m.mu.TryLock() { + t.Fatal("host call holds manager mutex") + } + m.mu.Unlock() + if !sb.Transition.TryLock() { + t.Fatal("host call holds transition mutex") + } + sb.Transition.Unlock() + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatal(err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("writer overtook canceled detach worker: %v", err) + } + close(eng.resume) + resumed = true + if err := <-changed; err == nil { + t.Fatal("released claim consumed late detach as successful mutation") + } + m.retryRemovals(t.Context()).Wait() + m.eng = base + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatal(err) + } + m.retryRemovals(t.Context()).Wait() + if !volumeDirty(catalog[0].Path) { + t.Fatal("old cleanup erased new writer marker") + } +} + +type gatedDetachEngine struct { + *fakeEngine + entered chan struct{} + resume chan struct{} +} + +func (e *gatedDetachEngine) DetachVolume(ctx context.Context, vm, sock, name, mount string) (engine.VolumeDetachResult, error) { + result, err := e.fakeEngine.DetachVolume(ctx, vm, sock, name, mount) + close(e.entered) + <-e.resume + return result, err +} + +func TestLiveVolumeModesRetryFailedIntentAndAttachWrites(t *testing.T) { + for _, stage := range []string{"intent", "attached"} { + for _, v := range []types.Volume{{Name: "data"}, {Name: "data", Mode: types.VolumeModeRW}, {Name: "data", AttachOnly: true}, {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} { + t.Run(fmt.Sprintf("%s/rw=%t/raw=%t", stage, v.RW(), v.AttachOnly), func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + m.store.recommitting.Store(true) + fail := func([]byte) error { return errors.New("claim write unavailable") } + if stage == "intent" { + m.store.replaceFile = fail + } else { + eng.liveVolumeHook = func(verb string) error { + if verb == "attach" { + m.store.replaceFile = fail + } + return nil + } + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{v}); err == nil { + t.Fatal("failed claim write reported success") + } + if len(sb.VolumesWant) != 1 || sb.VolumesWant[0].AttachOnly != v.AttachOnly { + t.Fatal("failed write lost per-entry intent") + } + if stage == "intent" && len(eng.volumeSpecs) != 0 { + t.Fatal("failed intent reached device call") + } + eng.liveVolumeHook = nil + m.store.replaceFile = nil + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{v}); err != nil { + t.Fatalf("same target failed to recover: %v", err) + } + assertLiveJournal(t, m, sb) + holds := volumeHoldersOf(m, "data") + if holds.readers+holds.writers != 1 { + t.Fatalf("retry duplicated hold: %+v", holds) + } + }) + } + } +} + +func TestLiveVolumeModesCanceledAttachRetainsRetryableIntent(t *testing.T) { + for _, v := range []types.Volume{{Name: "data"}, {Name: "data", Mode: types.VolumeModeRW}, {Name: "data", AttachOnly: true}, {Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} { + t.Run(fmt.Sprintf("rw=%t/raw=%t", v.RW(), v.AttachOnly), func(t *testing.T) { + base := newFakeEngine() + m := newVolumeManager(t, base, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + eng := newGatedVolumeEngine(base) + m.eng = eng + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { _, err := m.SetVolumes(ctx, sb.ID, "", []types.Volume{v}); done <- err }() + <-eng.entered + cancel() + close(eng.resume) + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("canceled attachment outcome: %v", err) + } + if len(sb.VolumesWant) != 1 || volumeHoldersOf(m, "data").readers+volumeHoldersOf(m, "data").writers != 1 { + t.Fatal("canceled attachment dropped intent or hold") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{v}); err != nil { + t.Fatalf("canceled request retry: %v", err) + } + assertLiveJournal(t, m, sb) + }) + } +} diff --git a/sandboxd/pool/volume_mutation.go b/sandboxd/pool/volume_mutation.go index 33dc50e2..f973573b 100644 --- a/sandboxd/pool/volume_mutation.go +++ b/sandboxd/pool/volume_mutation.go @@ -7,6 +7,8 @@ import ( "maps" "slices" + "github.com/projecteru2/core/log" + "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -16,12 +18,13 @@ const ( volumeChangeInterrupted = "volume change interrupted; retry the desired target" ) -type volumeEdit func([]types.Volume) ([]types.Volume, error) +type volumeEdit func(current, shapeContext []types.Volume) ([]types.Volume, error) type volumeRun struct { cancel context.CancelFunc done chan struct{} holds []types.Volume + access volumeAccess } // GetVolumes reads a claim's desired and applied volumes; a tenant reaches only its own claims. @@ -37,16 +40,16 @@ func (m *Manager) GetVolumes(id, tenant string) (types.SandboxVolumesResponse, e // SetVolumes converges a running claim onto the full desired volume set. func (m *Manager) SetVolumes(ctx context.Context, id, tenant string, want []types.Volume) (types.SandboxVolumesResponse, error) { - return m.changeVolumes(ctx, id, tenant, func([]types.Volume) ([]types.Volume, error) { return slices.Clone(want), nil }) + return m.changeVolumes(ctx, id, tenant, func([]types.Volume, []types.Volume) ([]types.Volume, error) { return slices.Clone(want), nil }) } -// PatchVolumes adds and removes a running claim's read-only mounted volumes by name. +// PatchVolumes adds and removes a running claim's volumes by name. func (m *Manager) PatchVolumes(ctx context.Context, id, tenant string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { - return m.changeVolumes(ctx, id, tenant, func(current []types.Volume) ([]types.Volume, error) { + return m.changeVolumes(ctx, id, tenant, func(current, shapeContext []types.Volume) ([]types.Volume, error) { if err := validateDetachNames(patch.Remove); err != nil { return nil, err } - additions, err := normalizeLiveVolumes(patch.Add, current) + additions, err := normalizeLiveVolumes(patch.Add, shapeContext) if err != nil { return nil, err } @@ -73,7 +76,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v } var run *volumeRun if err = m.eng.CheckLiveVolumeSupport(ctx); err == nil { - ctx, run, err = m.beginVolumeRun(ctx, sb) + ctx, run, err = m.beginVolumeRun(ctx, sb, tenant) } if err != nil { view, _ = m.GetVolumes(id, tenant) @@ -86,34 +89,39 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v }() m.mu.Lock() current, held := slices.Clone(sb.VolumesWant), heldVolumes(sb) + recovery := m.store.recoveryVolumes(sb.ID)[sb.ID] m.mu.Unlock() - target, err := change(current) + shapeContext := slices.Concat(held, recovery) + target, err := change(current, shapeContext) if err != nil { return view, err } - target, err = normalizeLiveVolumes(target, held) + target, err = normalizeLiveVolumes(target, shapeContext) if err != nil { return view, err } - if err = validateLiveTarget(held, target); err != nil { + if err = validateLiveTarget(shapeContext, target); err != nil { return view, err } - additions := volumeDifference(target, held) - if len(additions) != 0 { + additions := volumeDifferenceByName(target, held) + if len(additions) > 0 { if err = m.tenantRemoved(ctx, sb.Tenant); err != nil { return view, err } } - resolved, err := m.resolveVolumes(ctx, sb.Tenant, additions) + resolved, err := m.resolveVolumesFor(ctx, run.access, additions) if err != nil { return view, err } if err = m.checkVolumeRun(ctx, sb, run); err != nil { return view, err } + if err = m.checkLiveAdditions(sb, additions); err != nil { + return view, err + } sb.Touch() m.mu.Lock() - uncertain := volumeDifference(volumeDifference(sb.VolumesWant, sb.Volumes), target) + uncertain := volumeDifferenceByName(volumeDifferenceByName(sb.VolumesWant, sb.Volumes), target) m.mu.Unlock() for _, v := range uncertain { failedName = v.Name @@ -126,7 +134,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v return view, err } m.mu.Lock() - removals := volumeDifference(sb.Volumes, target) + removals := volumeDifferenceByName(sb.Volumes, target) m.mu.Unlock() for _, v := range removals { failedName = v.Name @@ -139,7 +147,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v m.clearVolumeError(sb, run, v.Name) } m.mu.Lock() - pending := volumeDifference(sb.VolumesWant, sb.Volumes) + pending := volumeDifferenceByName(sb.VolumesWant, sb.Volumes) m.mu.Unlock() for _, v := range pending { failedName = v.Name @@ -167,7 +175,7 @@ func (m *Manager) changeVolumes(ctx context.Context, id, tenant string, change v return view, nil } -func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (context.Context, *volumeRun, error) { +func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox, caller string) (context.Context, *volumeRun, error) { for { if err := ctx.Err(); err != nil { return ctx, nil, err @@ -188,7 +196,10 @@ func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (contex continue } runCtx, cancel := context.WithCancel(ctx) - run := &volumeRun{cancel: cancel, done: make(chan struct{}), holds: heldVolumes(sb)} + run := &volumeRun{ + cancel: cancel, done: make(chan struct{}), holds: heldVolumes(sb), + access: volumeAccessFor(caller, sb.Tenant), + } m.volumeRuns[sb] = run m.mu.Unlock() return runCtx, run, nil @@ -198,7 +209,7 @@ func (m *Manager) beginVolumeRun(ctx context.Context, sb *types.Sandbox) (contex func (m *Manager) finishVolumeRun(sb *types.Sandbox, run *volumeRun) types.SandboxVolumesResponse { m.mu.Lock() if m.claimed[sb.ID] == sb { - m.releaseVolumes(volumeDifference(run.holds, heldVolumes(sb))) + m.releaseVolumes(volumeDifferenceByName(run.holds, heldVolumes(sb))) } delete(m.volumeRuns, sb) view := m.volumeView(sb) @@ -210,7 +221,7 @@ func (m *Manager) finishVolumeRun(sb *types.Sandbox, run *volumeRun) types.Sandb func (m *Manager) restoreVolumeHolds(sb *types.Sandbox, holds []types.Volume) { if m.volumeRuns[sb] == nil { - m.releaseVolumes(volumeDifference(holds, heldVolumes(sb))) + m.releaseVolumes(volumeDifferenceByName(holds, heldVolumes(sb))) } } @@ -256,24 +267,12 @@ func (m *Manager) prepareVolumeTarget(ctx context.Context, sb *types.Sandbox, ru } func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) (claimSnapshot, bool, error) { - sb.Transition.Lock() - defer sb.Transition.Unlock() - m.mu.Lock() - if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { - m.mu.Unlock() + if err := m.admitVolumeTarget(ctx, sb, run, target, additions, resolved); err != nil { return claimSnapshot{}, false, err } - if sb.OnExpire == types.ExpireArchive && len(target) > 0 { - m.mu.Unlock() - return claimSnapshot{}, false, ErrVolumeCapture - } - if err := m.reserveVolumes(additions); err != nil { - m.mu.Unlock() - return claimSnapshot{}, false, err - } - run.holds = append(run.holds, additions...) - m.mu.Unlock() cleanErr := m.confirmVolumesClean(resolved) + sb.Transition.Lock() + defer sb.Transition.Unlock() m.mu.Lock() defer m.mu.Unlock() if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { @@ -281,7 +280,7 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run } if cleanErr != nil { m.releaseVolumes(additions) - run.holds = volumeDifference(run.holds, additions) + run.holds = volumeDifferenceByName(run.holds, additions) return claimSnapshot{}, false, cleanErr } replaced := func(v types.Volume) bool { @@ -295,14 +294,82 @@ func (m *Manager) stageVolumeTarget(ctx context.Context, sb *types.Sandbox, run return m.store.set(sb), true, nil } +func (m *Manager) admitVolumeTarget(ctx context.Context, sb *types.Sandbox, run *volumeRun, target, additions []types.Volume, resolved []resolvedVolume) error { + sb.Transition.Lock() + defer sb.Transition.Unlock() + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { + return err + } + if sb.OnExpire == types.ExpireArchive && len(target) > 0 { + return ErrVolumeCapture + } + if slices.ContainsFunc(additions, types.Volume.RW) { + if err := m.liveWriterError(sb); err != nil { + return err + } + } + if err := m.reserveResolved(run.access, resolved, sb); err != nil { + return err + } + run.holds = append(run.holds, additions...) + return nil +} + +func (m *Manager) checkLiveAdditions(sb *types.Sandbox, additions []types.Volume) error { + m.mu.Lock() + defer m.mu.Unlock() + if slices.ContainsFunc(additions, types.Volume.RW) { + return m.liveWriterError(sb) + } + return nil +} + +// liveWriterError checks the policy and recoverable replicas of this claim; callers hold m.mu. +func (m *Manager) liveWriterError(sb *types.Sandbox) error { + if m.view.Load().poolReplicate[sb.PolicyKey()] > 0 { + return ErrReplicatingPool + } + if m.archives == nil && (sb.HasReplica() || sb.ArchivePending != "") { + return ErrReplicaStranded + } + if sb.HasReplica() || len(m.store.checkpointRefs(sb.ID)) > 0 { + return ErrReplicaPresent + } + return nil +} + func (m *Manager) detachVolumeRun(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume) error { - if err := m.checkVolumeRun(ctx, sb, run); err != nil { + path, err := m.detachImagePath(ctx, sb, run, v.Name) + if err != nil { return err } - if err := m.eng.DetachVolume(ctx, sb.VMName, sb.VsockSocket, v.Name, v.Mount); err != nil { + result, err := m.eng.DetachVolume(ctx, sb.VMName, sb.VsockSocket, v.Name, v.Mount) + if err != nil { return fmt.Errorf("detach volume %q: %w", v.Name, err) } - return m.checkVolumeRun(ctx, sb, run) + if err = m.checkVolumeRun(ctx, sb, run); err != nil { + return err + } + if needsUnmount(v) && result.NormalUnmount && path != "" { + if err = clearVolumeDirty(path); err != nil { + log.WithFunc("pool.detachVolumeRun").Errorf(ctx, err, "clear dirty marker for volume %s", v.Name) + } + } + return nil +} + +func (m *Manager) detachImagePath(ctx context.Context, sb *types.Sandbox, run *volumeRun, name string) (string, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.checkVolumeRunLocked(ctx, sb, run); err != nil { + return "", err + } + if entry := m.catalog.entries[name]; entry != nil { + return entry.disk.Path, nil + } + return "", nil } func (m *Manager) commitVolumeApplied(ctx context.Context, sb *types.Sandbox, run *volumeRun, v types.Volume, attached bool) error { @@ -333,9 +400,9 @@ func (m *Manager) commitVolumeRun(ctx context.Context, sb *types.Sandbox, run *v if err := ctx.Err(); err != nil { return err } - dropped := volumeDifference(run.holds, heldVolumes(sb)) + dropped := volumeDifferenceByName(run.holds, heldVolumes(sb)) m.releaseVolumes(dropped) - run.holds = volumeDifference(run.holds, dropped) + run.holds = volumeDifferenceByName(run.holds, dropped) return nil } @@ -345,17 +412,20 @@ func (m *Manager) ownsRunLocked(sb *types.Sandbox, run *volumeRun) bool { func (m *Manager) hasVolumeHolds(sb *types.Sandbox) bool { run := m.volumeRuns[sb] - return hasVolumes(sb) || run != nil && len(run.holds) > 0 + return hasVolumes(sb) || run != nil && len(run.holds) > 0 || len(m.store.recoveryVolumes(sb.ID)[sb.ID]) > 0 } func (m *Manager) volumeView(sb *types.Sandbox) types.SandboxVolumesResponse { + held := heldVolumes(sb) + recovering := volumeDifferenceByName(m.store.recoveryVolumes(sb.ID)[sb.ID], held) view := types.SandboxVolumesResponse{ Volumes: slices.Clone(sb.Volumes), VolumesWant: slices.Clone(sb.VolumesWant), States: []types.VolumeState{}, - Converging: m.volumeRuns[sb] != nil, + Converging: m.volumeRuns[sb] != nil || len(recovering) > 0, } - for _, v := range heldVolumes(sb) { + held = append(held, recovering...) + for _, v := range held { state := types.VolumeAttached if volumeIndex(sb.VolumesWant, v.Name) < 0 { state = types.VolumeDetaching @@ -413,29 +483,14 @@ func (m *Manager) clearVolumeError(sb *types.Sandbox, run *volumeRun, name strin } func normalizeLiveVolumes(volumes, held []types.Volume) ([]types.Volume, error) { - if len(volumes) > types.MaxClaimVolumes { - return nil, fmt.Errorf("%w: too many volumes", ErrBadVolume) - } - out := make([]types.Volume, 0, len(volumes)) - for _, v := range volumes { - attachOnly := v.AttachOnly - if i := volumeIndex(held, v.Name); i >= 0 && held[i].Mount == "" && v.Mount == "" { - attachOnly = true - } - normalized, err := types.ValidateVolumes([]types.Volume{v}, attachOnly) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) - } - if volumeIndex(out, v.Name) >= 0 { - return nil, fmt.Errorf("%w: duplicate volume %q", ErrBadVolume, v.Name) - } - if attachOnly { - normalized[0].AttachOnly = false + request := slices.Clone(volumes) + for i, v := range request { + if old := volumeIndex(held, v.Name); old >= 0 && held[old].AttachOnly && v.Mount == "" { + request[i].AttachOnly = true } - out = append(out, normalized[0]) } - mounted := slices.DeleteFunc(slices.Clone(out), func(v types.Volume) bool { return v.Mount == "" }) - if _, err := types.ValidateVolumes(mounted, false); err != nil { + out, err := types.ValidateVolumes(request) + if err != nil { return nil, fmt.Errorf("%w: %w", ErrBadVolume, err) } return out, nil @@ -447,23 +502,15 @@ func validateLiveTarget(held, target []types.Volume) error { if i >= 0 && target[i] != v { return fmt.Errorf("%w: remove volume %q before changing its mount or mode", ErrBadVolume, v.Name) } - if immutableVolume(v) && i < 0 { - return fmt.Errorf("%w: writable and attach-only volumes cannot change live", ErrBadVolume) - } - } - if slices.ContainsFunc(volumeDifference(target, held), immutableVolume) { - return fmt.Errorf("%w: live additions require read-only mounts", ErrBadVolume) } return nil } -func immutableVolume(v types.Volume) bool { return v.RW() || v.AttachOnly || v.Mount == "" } - func volumeIndex(volumes []types.Volume, name string) int { return slices.IndexFunc(volumes, func(v types.Volume) bool { return v.Name == name }) } -func volumeDifference(left, right []types.Volume) []types.Volume { +func volumeDifferenceByName(left, right []types.Volume) []types.Volume { return slices.DeleteFunc(slices.Clone(left), func(v types.Volume) bool { return volumeIndex(right, v.Name) >= 0 }) } @@ -480,5 +527,5 @@ func validateDetachNames(names []string) error { } func heldVolumes(sb *types.Sandbox) []types.Volume { - return slices.Concat(sb.Volumes, volumeDifference(sb.VolumesWant, sb.Volumes)) + return slices.Concat(sb.Volumes, volumeDifferenceByName(sb.VolumesWant, sb.Volumes)) } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index ce30fd1a..f39f22bf 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -68,8 +68,8 @@ func TestLiveAttachPreservesWritableAndAttachOnlyVolumes(t *testing.T) { if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "other", Mount: original.Mount}}); !errors.Is(err, ErrBadVolume) { t.Fatalf("accepted mode change: %v", err) } - if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "other"}}); !errors.Is(err, ErrBadVolume) || len(sb.Volumes) != 2 { - t.Fatalf("writable detach changed the batch: %v", err) + if _, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data", "other"}}); err != nil || len(sb.Volumes) != 0 { + t.Fatalf("writable detach failed: %v", err) } }) } @@ -80,8 +80,8 @@ func TestLiveVolumeValidationPrecedesDeviceCalls(t *testing.T) { m := newVolumeManager(t, eng, liveCatalog(t)) sb := mustClaim(t, m, testKey) for _, request := range [][]types.Volume{ - {{Name: "data", Mode: "rw"}}, - {{Name: "data", AttachOnly: true}}, + {{Name: "data", Mode: "unknown"}}, + {{Name: "data", AttachOnly: true, Mount: "/data"}}, {{Name: "data", Mount: "/etc"}}, {{Name: "data"}, {Name: "data"}}, {{Name: "data", Mount: "/a"}, {Name: "other", Mount: "/a/b"}}, @@ -280,8 +280,11 @@ func TestLiveVolumeDoesNotBlockLifecycle(t *testing.T) { } close(resume) attachErr := <-attached - if action == "reap" { - waitFor(t, func() bool { return volumeHoldersOf(m, "data") == (volumeHolders{}) }) + if action == "release" || action == "reap" { + waitFor(t, func() bool { + m.retryRemovals(t.Context()).Wait() + return volumeHoldersOf(m, "data") == (volumeHolders{}) + }) } if action == "release" || action == "reap" { if attachErr == nil { @@ -718,6 +721,7 @@ func TestRollbackClaimTakesOverAdmittedRunHolds(t *testing.T) { run := beginAdmittedVolumeRun(t, m, sb) m.rollbackClaim(t.Context(), []*types.Sandbox{sb}) m.finishVolumeRun(sb, run) + m.retryRemovals(t.Context()).Wait() waitFor(t, m.store.synced) if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { t.Fatalf("rolled-back claim leaked its admitted hold: %+v", got) @@ -846,7 +850,7 @@ func assertLiveJournal(t *testing.T, m *Manager, sb *types.Sandbox) { func beginAdmittedVolumeRun(t *testing.T, m *Manager, sb *types.Sandbox) *volumeRun { t.Helper() - _, run, err := m.beginVolumeRun(t.Context(), sb) + _, run, err := m.beginVolumeRun(t.Context(), sb, sb.Tenant) if err != nil { t.Fatal(err) } diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go new file mode 100644 index 00000000..e723753a --- /dev/null +++ b/sandboxd/pool/volume_replica_test.go @@ -0,0 +1,395 @@ +package pool + +import ( + "errors" + "fmt" + "slices" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestReplicaRetirementKeepsTombstoneUntilClaimCommit(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + ck := sb.ReplicaCk + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected claims write failure") } + sb.Transition.Lock() + err := m.dropReplica(t.Context(), sb) + sb.Transition.Unlock() + if err == nil { + t.Error("retirement acknowledged a failed claim commit") + } + rec, ok, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !ok || !rec.Consumed || !ckExists(t, m, ck) { + t.Fatalf("failed commit lost recovery tombstone/checkpoint: %+v, exists=%t, err=%v", rec, ok, err) + } + m.reconcileOwnRow(t.Context(), rec, time.Now()) + if rec, ok, err = c.reg.Get(t.Context(), sb.ID); err != nil || !ok || !rec.Consumed { + t.Fatalf("sweep erased unacknowledged retirement: %+v, exists=%t, err=%v", rec, ok, err) + } + restarted := c.restart(t, m) + if err = restarted.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + restored, ok := restarted.byID(sb.ID) + if !ok { + t.Fatal("restart dropped a claim after failed retirement commit") + } + restarted.retireReplica(t.Context(), restored) + if restored.HasReplica() { + t.Fatal("restart did not finish retirement") + } +} + +func TestReplicaUnpublishedIntentSurvivesFailedClearAndRestart(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + m.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + pending := sb.ReplicaPending + if pending == "" || !ckExists(t, m, pending) { + t.Fatal("missing unpublished checkpoint") + } + m.archives = c.reg + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected pending clear failure") } + m.retireReplica(t.Context(), sb) + persisted, err := newClaimStore(m.dataDir, false).load() + if err != nil || persisted[sb.ID].ReplicaPending != pending || len(persisted[sb.ID].VolumesWant) != 0 { + t.Fatalf("failure did not retain old disk state: %+v, %v", persisted, err) + } + if !ckExists(t, m, pending) { + t.Error("failed pending clear deleted the checkpoint used to distinguish an unpublished intent from adoption") + } + t.Logf("failed unpublished clear: memory_pending=%q durable_pending=%q checkpoint_exists=%t wanted=%+v synced=%t", sb.ReplicaPending, persisted[sb.ID].ReplicaPending, ckExists(t, m, pending), persisted[sb.ID].VolumesWant, m.store.synced()) + restarted := restartReplicaVolumeManager(t, m, c) + restored, ok := restarted.byID(sb.ID) + if !ok { + t.Fatal("restart mistook a failed pending clear for adoption and discarded the claim") + } + if len(restored.VolumesWant) != 0 || len(restored.Volumes) != 0 { + t.Fatalf("restart lost pending writer: %+v", restored) + } + restarted.retireReplica(t.Context(), restored) + if restored.HasReplica() { + t.Fatal("restart failed to retire unpublished replica") + } +} + +func TestReplicaPendingCheckpointCleanupWaitsForAcknowledgment(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + m.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + pending := sb.ReplicaPending + m.archives = c.reg + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected pending clear failure") } + m.retireReplica(t.Context(), sb) + if sb.ReplicaPending != "" || m.store.synced() { + t.Fatal("expected memory clear before snapshot acknowledgment") + } + if deleted, err := m.deleteArchiveCk(t.Context(), pending); err != nil || deleted { + t.Fatalf("orphan delete ignored unacknowledged claim: deleted=%t, %v", deleted, err) + } + m.retryArchiveDeletes(t.Context()) + if !ckExists(t, m, pending) || !m.deleteMarked(pending) { + t.Fatal("cleanup discarded pending checkpoint or retry marker") + } + t.Logf("cleanup deferred: checkpoint=%s exists=%t marker=%t synced=%t", pending, ckExists(t, m, pending), m.deleteMarked(pending), m.store.synced()) + m.store.replaceFile = nil + if err := m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + other := mustClaim(t, m, testKey) + m.mu.Lock() + other.LeaseSeconds++ + m.store.set(other) + m.mu.Unlock() + m.retryArchiveDeletes(t.Context()) + if ckExists(t, m, pending) || m.deleteMarked(pending) { + t.Fatal("acknowledged cleanup leaked checkpoint or retry marker") + } + t.Logf("cleanup acknowledged: exists=%t marker=%t synced=%t", ckExists(t, m, pending), m.deleteMarked(pending), m.store.synced()) +} + +func TestReplicaSettledCheckpointCleanupRetriesAfterFailedCommit(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + previous := sb.ReplicaCk + m.archives = &hookRegistry{Registry: c.reg, losePutReply: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + pending := sb.ReplicaPending + m.archives = c.reg + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected settled intent failure") } + sb.Transition.Lock() + err := m.settleReplicaIntent(t.Context(), sb) + sb.Transition.Unlock() + if err == nil || sb.ReplicaPending != "" || sb.ReplicaCk != pending || m.store.synced() { + t.Fatalf("settle state: %+v, %v", sb, err) + } + if !ckExists(t, m, previous) || !m.deleteMarked(previous) { + t.Fatal("failed settle lost old checkpoint or its queued cleanup") + } + m.retryArchiveDeletes(t.Context()) + if !ckExists(t, m, previous) { + t.Fatal("old checkpoint deleted before clear acknowledgment") + } + m.store.replaceFile = nil + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + m.retryArchiveDeletes(t.Context()) + if ckExists(t, m, previous) || m.deleteMarked(previous) || !ckExists(t, m, pending) { + t.Fatal("acknowledged settle failed to clean old checkpoint or removed current checkpoint") + } +} + +func TestReplicaRecoveryDoesNotRepublishReadOnlyHold(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + if view, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil || len(view.Volumes) != 1 { + t.Fatalf("read-only live attachment: %+v, %v", view, err) + } + if _, err := c.reg.Consume(t.Context(), sb.ID, sb.ArchiveEpoch); err != nil { + t.Fatal(err) + } + rec, ok, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !ok || !rec.Consumed { + t.Fatalf("setup: %+v, %t, %v", rec, ok, err) + } + m.reconcileOwnRow(t.Context(), rec, time.Now()) + if rec, ok, err = c.reg.Get(t.Context(), sb.ID); err != nil || !ok || !rec.Consumed { + t.Fatalf("recovery republished read-only hold: %+v, %t, %v", rec, ok, err) + } +} + +func TestReplicaConsumedRowPurgeWaitsForSnapshot(t *testing.T) { + for _, claimed := range []bool{false, true} { + t.Run(fmt.Sprintf("claimed=%t", claimed), func(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected clear failure") } + m.retireReplica(t.Context(), sb) + if !claimed { + m.mu.Lock() + delete(m.claimed, sb.ID) + m.store.del(sb.ID) + m.mu.Unlock() + } + rec, ok, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !ok || !rec.Consumed { + t.Fatalf("setup: %+v, %t, %v", rec, ok, err) + } + m.reconcileOwnRow(t.Context(), rec, time.Now()) + if _, ok, err = c.reg.Get(t.Context(), sb.ID); err != nil || !ok { + t.Fatalf("sweep dropped unacknowledged row: %t, %v", ok, err) + } + m.store.replaceFile = nil + if err = m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + m.reconcileOwnRow(t.Context(), rec, time.Now()) + if _, ok, err = c.reg.Get(t.Context(), sb.ID); err != nil || ok { + t.Fatalf("sweep retained acknowledged row: %t, %v", ok, err) + } + }) + } +} + +func TestLiveWriterRefusesExistingReplica(t *testing.T) { + for _, pending := range []bool{false, true} { + for _, attachOnly := range []bool{false, true} { + for _, patch := range []bool{false, true} { + t.Run(fmt.Sprintf("pending=%t/attach_only=%t/patch=%t", pending, attachOnly, patch), func(t *testing.T) { + m, c := newReplicaVolumeManager(t) + disableReplication(m) + sb := mustClaim(t, m, testKey) + if pending { + m.archives = &hookRegistry{Registry: c.reg, losePutReply: true} + if err := m.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatal(err) + } + } else { + replicateNow(t, m, sb) + } + row, exists, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !exists || row.Consumed || !sb.HasReplica() { + t.Fatalf("replica setup failed: %+v, %t, %v", row, exists, err) + } + eng := m.eng.(*fakeEngine) + before := eng.volumeOpsLog() + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: attachOnly}} + var view types.SandboxVolumesResponse + if patch { + view, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: writer}) + } else { + view, err = m.SetVolumes(t.Context(), sb.ID, "", writer) + } + if !errors.Is(err, ErrReplicaPresent) { + t.Fatalf("writer admitted while replica is recoverable: %v", err) + } + if len(view.Volumes) != 0 || len(view.VolumesWant) != 0 || view.Converging || !slices.Equal(before, eng.volumeOpsLog()) || volumeHoldersOf(m, "data").writers != 0 { + t.Fatalf("refused writer changed devices or holds: %+v", view) + } + if next, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || !ok || next.Checkpoint != row.Checkpoint || next.Consumed { + t.Fatalf("refusal changed replica: %+v, %t, %v", next, ok, getErr) + } + m.archives = c.reg + m.retireReplica(t.Context(), sb) + if sb.HasReplica() { + t.Fatal("retirement did not settle the replica") + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", writer); err != nil { + t.Fatalf("writer after replica retirement: %v", err) + } + }) + } + } + } +} + +func TestReplicaPublicationRefusesVolumeAdmittedDuringExport(t *testing.T) { + m, c := newReplicaVolumeManager(t) + disableReplication(m) + sb := mustClaim(t, m, testKey) + stalled := &stallingStore{Store: m.ckpts, published: make(chan string), release: make(chan struct{})} + m.ckpts = stalled + result := make(chan error, 1) + go func() { result <- m.replicate(t.Context(), sb) }() + ck := <-stalled.published + release := sync.OnceFunc(func() { close(stalled.release) }) + t.Cleanup(release) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + view, err := m.SetVolumes(t.Context(), sb.ID, "", writer) + if err != nil || len(view.Volumes) != 1 { + t.Fatalf("writer during checkpoint publication: %+v, %v", view, err) + } + release() + if err = <-result; !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("stale replica registered after writer admission: %v", err) + } + if _, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || ok || sb.HasReplica() || ckExists(t, m, ck) { + t.Fatalf("stale replica remains recoverable: row=%t, claim=%t, checkpoint=%t, err=%v", ok, sb.HasReplica(), ckExists(t, m, ck), getErr) + } + if got := volumeHoldersOf(m, "data"); got.writers != 1 { + t.Fatalf("refusing replica lost writer hold: %+v", got) + } + if err = m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("snapshot admission while writer is held: %v", err) + } +} + +func TestReplicaCaptureRefusesPendingLiveWriter(t *testing.T) { + m, c := newReplicaVolumeManager(t) + disableReplication(m) + sb := mustClaim(t, m, testKey) + eng := m.eng.(*fakeEngine) + entered, resume := make(chan struct{}), make(chan struct{}) + release := sync.OnceFunc(func() { close(resume) }) + t.Cleanup(release) + eng.liveVolumeHook = func(verb string) error { + if verb == "attach" { + close(entered) + <-resume + } + return nil + } + result := make(chan error, 1) + go func() { + _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}) + result <- err + }() + <-entered + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 1 || !view.Converging { + t.Fatalf("writer did not reach pending attachment: %+v, %v", view, err) + } + if err = m.replicate(t.Context(), sb); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("captured pending writer: %v", err) + } + if _, ok, getErr := c.reg.Get(t.Context(), sb.ID); getErr != nil || ok || len(eng.snapSaves) != 0 { + t.Fatalf("pending writer produced snapshot or row: snapshots=%v, row=%t, err=%v", eng.snapSaves, ok, getErr) + } + release() + if err = <-result; err != nil { + t.Fatalf("pending writer did not complete: %v", err) + } +} + +func TestReplicaAllowsLiveReadOnlyAndCleanup(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + reader := []types.Volume{{Name: "data"}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", reader); err != nil { + t.Fatalf("reader with replica: %v", err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", reader); err != nil { + t.Fatalf("held reader retry: %v", err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("reader detach with replica: %v", err) + } +} + +func newReplicaVolumeManager(t *testing.T) (*Manager, *failoverCluster) { + t.Helper() + c := newReplicaCluster(t) + spec := archivePool(keepAnHour) + spec.ReplicateSeconds = c.replicate + cfg := &config.Config{ + APIToken: "root", DataDir: t.TempDir(), CheckpointDir: c.ckDir, + Pools: []config.PoolSpec{spec}, Mesh: &config.MeshConfig{NodeID: "a"}, + Volumes: []config.VolumeSpec{{Name: "data", Path: writeVolumeImage(t, "data", "original"), Writable: true}}, + } + backends := testBackends(t, cfg) + backends.Archives = c.reg + m, err := NewManager(t.Context(), cfg, newFakeEngine(), testSecrets(t), backends) + if err != nil { + t.Fatal(err) + } + c.renew(t, "a") + m.liftFence() + return m, c +} + +func restartReplicaVolumeManager(t *testing.T, old *Manager, c *failoverCluster) *Manager { + t.Helper() + spec := archivePool(keepAnHour) + cfg := &config.Config{APIToken: "root", DataDir: old.dataDir, CheckpointDir: c.ckDir, Pools: []config.PoolSpec{spec}, Mesh: &config.MeshConfig{NodeID: old.nodeID}, Volumes: []config.VolumeSpec{{Name: "data", Path: old.catalog.entries["data"].disk.Path, Writable: true}}} + backends := testBackends(t, cfg) + backends.Archives = c.reg + m, err := NewManager(t.Context(), cfg, old.eng, testSecrets(t), backends) + if err != nil { + t.Fatal(err) + } + if err = m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + m.liftFence() + return m +} + +func disableReplication(m *Manager) { + view := *m.view.Load() + view.poolReplicate = nil + m.view.Store(&view) +} diff --git a/sandboxd/pool/volume_replication_policy_test.go b/sandboxd/pool/volume_replication_policy_test.go new file mode 100644 index 00000000..d2a66f44 --- /dev/null +++ b/sandboxd/pool/volume_replication_policy_test.go @@ -0,0 +1,259 @@ +package pool + +import ( + "errors" + "fmt" + "slices" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/store" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestLiveWriterDoesNotWaitForAnotherClaimsRenewal(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + other := mustClaim(t, m, testKey) + attached := make(chan struct{}, 1) + eng.liveVolumeHook = func(op string) error { + if op == "attach" { + attached <- struct{}{} + } + return nil + } + resume := make(chan struct{}) + var once sync.Once + release := func() { once.Do(func() { close(resume) }) } + defer release() + renewed := make(chan error, 1) + var writes atomic.Int32 + m.store.replaceFile = func(raw []byte) error { + switch writes.Add(1) { + case 1: + if err := m.store.replace(raw); err != nil { + return err + } + before := m.store.mark().seq + go func() { + _, err := m.Renew(t.Context(), other.ID, Cred{Token: other.Token}, time.Hour, "") + renewed <- err + }() + deadline := time.Now().Add(3 * time.Second) + for m.store.mark().seq == before { + if time.Now().After(deadline) { + return errors.New("renew did not stage its unrelated claim") + } + time.Sleep(time.Millisecond) + } + return nil + case 2: + <-resume + } + return m.store.replace(raw) + } + changed := make(chan error, 1) + go func() { + _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}}) + changed <- err + }() + finished := false + select { + case <-attached: + case err := <-changed: + finished = true + t.Errorf("unrelated renewal rejected writable attach: %v", err) + case <-time.After(3 * time.Second): + t.Error("unrelated renewal prevented writable device application") + } + release() + if !finished { + if err := <-changed; err != nil { + t.Errorf("writable attach after renewal: %v", err) + } + } + if err := <-renewed; err != nil { + t.Fatal(err) + } + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.Volumes) != 1 { + t.Fatalf("writer did not attach: %+v, %v", view, err) + } +} + +func TestArchiveCheckpointCleanupIgnoresUnrelatedClaimWrites(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + checkpoint, _, err := m.publishCheckpoint(t.Context(), sb, store.CheckpointID(randHex(8)), "orphan", "", true) + if err != nil { + t.Fatal(err) + } + ck := checkpoint.ID + other := mustClaim(t, m, testKey) + m.mu.Lock() + other.LeaseSeconds++ + m.store.set(other) + m.mu.Unlock() + if m.store.synced() { + t.Fatal("expected an unrelated unacknowledged claim write") + } + if deleted, err := m.deleteArchiveCk(t.Context(), ck); err != nil || !deleted { + t.Fatalf("unrelated claim delayed checkpoint cleanup: deleted=%t, %v", deleted, err) + } + if ckExists(t, m, ck) || m.deleteMarked(ck) { + t.Fatal("unreferenced checkpoint or delete marker remained") + } +} + +func TestLiveWriterRequiresNonReplicatingPool(t *testing.T) { + for _, patch := range []bool{false, true} { + for _, attachOnly := range []bool{false, true} { + t.Run(fmt.Sprintf("patch=%t/attach_only=%t", patch, attachOnly), func(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + before := m.store.mark() + beforeOps := m.eng.(*fakeEngine).volumeOpsLog() + t.Logf("before live mutation: snapshot=%+v device_ops=%v", before, beforeOps) + want := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: attachOnly}} + var view types.SandboxVolumesResponse + var err error + if patch { + view, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: want}) + } else { + view, err = m.SetVolumes(t.Context(), sb.ID, "", want) + } + if !errors.Is(err, ErrReplicatingPool) { + t.Fatalf("replicating pool accepted live writer: %+v, %v", view, err) + } + if sb.HasReplica() || len(view.VolumesWant) != 0 || len(view.Volumes) != 0 || view.Converging || m.store.mark() != before || volumeHoldersOf(m, "data") != (volumeHolders{}) || !slices.Equal(beforeOps, m.eng.(*fakeEngine).volumeOpsLog()) { + t.Fatalf("policy refusal changed claim, device, or reservation: %+v snapshot=%+v->%+v ops=%v->%v", view, before, m.store.mark(), beforeOps, m.eng.(*fakeEngine).volumeOpsLog()) + } + }) + } + } +} + +func TestInitialWriterRemainsAvailableInReplicatingPool(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}) + if err != nil || len(sb.Volumes) != 1 { + t.Fatalf("initial writer: %+v, %v", sb, err) + } + m.replicateOnce(t.Context()) + waitFor(t, func() bool { return !m.replicating.Load() }) + if sb.HasReplica() || len(m.eng.(*fakeEngine).snapSaves) != 0 { + t.Fatal("replicated an initial writer") + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", sb.Volumes); err != nil { + t.Fatalf("held writer retry refused: %v", err) + } + if _, err = m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("writer cleanup refused: %v", err) + } +} + +func TestLiveWriterUsesItsPolicySource(t *testing.T) { + for _, sourceReplicates := range []bool{false, true} { + t.Run(fmt.Sprint(sourceReplicates), func(t *testing.T) { + m, _ := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + other := testKey + other.Template = "other:template" + m.mu.Lock() + if sourceReplicates { + sb.Key, sb.PolicySource = other, testKey + } else { + sb.PolicySource = other + } + m.mu.Unlock() + _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}) + if sourceReplicates && !errors.Is(err, ErrReplicatingPool) || !sourceReplicates && err != nil { + t.Fatalf("policy source replication=%t: %v", sourceReplicates, err) + } + }) + } +} + +func TestLiveWriterWaitsOnlyForItsOwnCheckpointReferences(t *testing.T) { + m, c := newReplicaVolumeManager(t) + sb := mustClaim(t, m, testKey) + other := mustClaim(t, m, testKey) + replicateNow(t, m, sb) + replicateNow(t, m, other) + m = restartReplicaVolumeManager(t, m, c) + sb, _ = m.byID(sb.ID) + writer := []types.Volume{{Name: "data", Mode: types.VolumeModeRW, AttachOnly: true}} + if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) { + t.Fatalf("restart ignored actual replica: %v", err) + } + m.store.recommitting.Store(true) + m.store.replaceFile = func([]byte) error { return errors.New("injected clear failure") } + m.retireReplica(t.Context(), sb) + before := m.store.mark() + if sb.HasReplica() || len(m.store.checkpointRefs(sb.ID)) == 0 { + t.Fatal("expected an acknowledged replica reference after memory clear") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", writer); !errors.Is(err, ErrReplicaPresent) || m.store.mark() != before { + t.Fatalf("unacknowledged replica clear allowed writer or staged intent: %v", err) + } + m.store.replaceFile = nil + if err := m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + if len(m.store.checkpointRefs(other.ID)) == 0 { + t.Fatal("unrelated replica did not survive the restart") + } + if view, err := m.SetVolumes(t.Context(), sb.ID, "", writer); err != nil || len(view.Volumes) != 1 { + t.Fatalf("acknowledged retirement blocked writer: %+v, %v", view, err) + } +} + +func TestAdmittedWriterRetryAfterRestartCanRemoveReader(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + eng.liveVolumeHook = func(op string) error { + if op == "attach" { + return errors.New("injected writable attachment failure") + } + return nil + } + writer := types.Volume{Name: "other", Mode: types.VolumeModeRW, AttachOnly: true} + view, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{writer}}) + if err == nil || len(view.VolumesWant) != 2 || len(view.Volumes) != 1 { + t.Fatalf("attachment failure did not retain admitted target: %+v, %v", view, err) + } + eng.liveVolumeHook = nil + c := newReplicaCluster(t) + spec := archivePool(keepAnHour) + spec.ReplicateSeconds = c.replicate + cfg := &config.Config{APIToken: "root", DataDir: m.dataDir, CheckpointDir: c.ckDir, Volumes: catalog, Pools: []config.PoolSpec{spec}, Mesh: &config.MeshConfig{NodeID: "a"}} + backends := testBackends(t, cfg) + backends.Archives = c.reg + m, err = NewManager(t.Context(), cfg, eng, testSecrets(t), backends) + if err != nil { + t.Fatal(err) + } + c.renew(t, "a") + if err = m.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + m.liftFence() + view, err = m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Remove: []string{"data"}}) + if err != nil || !slices.Equal(view.VolumesWant, []types.Volume{writer}) || !slices.Equal(view.Volumes, []types.Volume{writer}) { + t.Fatalf("admitted writer retry prevented reader removal: %+v, %v", view, err) + } + m.replicateOnce(t.Context()) + waitFor(t, func() bool { return !m.replicating.Load() }) + if rec, ok, err := c.reg.Get(t.Context(), sb.ID); err != nil || ok { + t.Fatalf("admitted writer acquired a replica row: %+v, exists=%t, %v", rec, ok, err) + } +} diff --git a/sandboxd/pool/volume_retirement_test.go b/sandboxd/pool/volume_retirement_test.go new file mode 100644 index 00000000..7dd90ce3 --- /dev/null +++ b/sandboxd/pool/volume_retirement_test.go @@ -0,0 +1,321 @@ +package pool + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestDetachedVolumeRetainsRecoveryProtectionUntilCommit(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, sb.Tenant, []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + view, err := m.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}) + eng.liveVolumeHook = nil + if err == nil || len(view.Volumes) != 0 || len(view.VolumesWant) != 0 { + t.Fatalf("detached result = %+v, %v", view, err) + } + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil || claims[sb.ID] == nil || len(claims[sb.ID].Volumes) != 1 { + t.Fatalf("failed write did not leave old recovery reference: %+v, %v", claims, err) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Errorf("writer admitted while old claim still references detached volume: %v", err) + } + if _, err = m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "unretired", ""); !errors.Is(err, ErrVolumeCapture) { + t.Errorf("capture passed before recovery reference retired: %v", err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err = m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("restart lost old claim's reader exclusion: %v", err) + } + if _, err = m2.PatchVolumes(t.Context(), sb.ID, sb.Tenant, types.VolumePatch{Remove: []string{"data"}}); err != nil { + t.Fatalf("retry after restart: %v", err) + } + if _, err = m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("writer still blocked after successful retirement: %v", err) + } +} + +func TestDetachedVolumeOwnRetryAndLaterAcknowledgment(t *testing.T) { + for _, retry := range []string{"empty", "same-shape", "unrelated-snapshot"} { + t.Run(retry, func(t *testing.T) { + eng := newFakeEngine() + m := newVolumeManager(t, eng, liveCatalog(t)) + sb := mustClaim(t, m, testKey) + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("detach result unexpectedly persisted") + } + eng.liveVolumeHook = nil + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("physical count retained after detach: %+v", got) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("empty retry acknowledged an unretired file") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data", Mount: "/different"}}); !errors.Is(err, ErrBadVolume) { + t.Fatalf("recovery-only shape changed: %v", err) + } + view, err := m.GetVolumes(sb.ID, "") + if err != nil || len(view.States) != 1 || view.States[0].State != types.VolumeDetaching { + t.Fatalf("unretired view: %+v, %v", view, err) + } + if err = os.Remove(tmp); err != nil { + t.Fatal(err) + } + switch retry { + case "empty": + _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + case "same-shape": + for range 2 { + if _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + if got := volumeHoldersOf(m, "data"); got.readers != 1 || got.writers != 0 { + t.Fatalf("own retry counts: %+v", got) + } + } + _, err = m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}) + case "unrelated-snapshot": + _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) + } + if err != nil { + t.Fatal(err) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("retired count: %+v", got) + } + if _, err = m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); err != nil { + t.Fatalf("writer after acknowledgment: %v", err) + } + }) + } +} + +func TestFailedClaimIntroductionAfterRenameRemainsProtected(t *testing.T) { + eng := newFakeEngine() + catalog := liveCatalog(t) + m := newVolumeManager(t, eng, catalog) + sb := mustClaim(t, m, testKey) + m.store.sync = true + m.store.recommitting.Store(true) + calls := 0 + m.store.replaceFile = func(raw []byte) error { + calls++ + if calls == 1 { + if err := m.store.replace(raw); err != nil { + return err + } + return errors.New("directory sync failed after rename") + } + return errors.New("replacement failed before rename") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); err == nil { + t.Fatal("expected post-rename intent error") + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{}); err == nil { + t.Fatal("expected empty snapshot error") + } + if len(sb.Volumes) != 0 || len(sb.VolumesWant) != 0 { + t.Fatalf("memory not empty: %+v", sb) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("unexpected physical count: %+v", got) + } + claims, err := newClaimStore(m.dataDir, false).load() + if err != nil || len(claims[sb.ID].VolumesWant) != 1 { + t.Fatalf("actual recovery file: %+v, %v", claims, err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("writer admitted after uncertain introduction: %v", err) + } + if _, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "uncertain-introduction", ""); !errors.Is(err, ErrVolumeCapture) { + t.Fatalf("capture admitted: %v", err) + } + m2 := newVolumeManagerAt(t, eng, m.dataDir, catalog) + if err := m2.Reconcile(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := m2.ClaimProvision(t.Context(), testKey, ClaimOptions{Volumes: []types.Volume{{Name: "data", Mode: types.VolumeModeRW}}}); !errors.Is(err, ErrVolumeBusy) { + t.Fatalf("restart admitted writer: %v", err) + } + m.store.replaceFile = nil + if err := m.store.commit(m.store.mark()); err != nil { + t.Fatal(err) + } + if len(m.store.recoveryVolumes()[sb.ID]) != 0 { + t.Fatal("successful whole snapshot did not retire attempted row") + } +} + +func TestClaimRecoveryReferencesSurviveResetAndAreCopied(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + s = newClaimStore(filepath.Dir(s.path), false) + if _, err := s.load(); err != nil { + t.Fatal(err) + } + s.reset(map[string]*types.Sandbox{}) + refs := s.recoveryVolumes() + refs[sb.ID][0].Name = "caller-copy" + if got := s.recoveryVolumes()[sb.ID]; len(got) != 1 || got[0].Name != "data" { + t.Fatalf("reset or caller erased actual file: %+v", got) + } + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if len(s.recoveryVolumes()) != 0 { + t.Fatal("empty snapshot did not retire file references") + } +} + +func TestClaimRecoveryAttemptsRetainOnlyDistinctReferences(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data", Mount: "/data"}}} + if err := s.commit(s.set(sb)); err != nil { + t.Fatal(err) + } + s.replaceFile = func([]byte) error { return errors.New("no space left") } + for i := range 200 { + sb.Token = fmt.Sprint(i) + sb.Volumes[0].Mount = []string{"/data", "/other"}[i%2] + if err := s.commit(s.set(sb)); err == nil { + t.Fatal("expected failed snapshot") + } + } + if len(s.attemptRefs) != 1 || len(s.attemptRefs[sb.ID].volumes) != 2 { + t.Fatalf("renewals retained duplicate history: %+v", s.attemptRefs) + } + refs := s.recoveryVolumes(sb.ID)[sb.ID] + refs[0].Name = "caller-copy" + if got := s.recoveryVolumes(sb.ID)[sb.ID]; len(got) != 2 || got[0].Name != "data" { + t.Fatalf("recovery references not copied: %+v", got) + } + s.replaceFile = nil + if err := s.commit(s.mark()); err != nil { + t.Fatal(err) + } + if len(s.attemptRefs) != 0 || len(s.recoveryVolumes(sb.ID)[sb.ID]) != 1 { + t.Fatal("successful snapshot did not replace possibility history") + } +} + +func TestRecoveryOnlyRetryRequiresNewAuthorization(t *testing.T) { + eng := newFakeEngine() + m, sb := newTenantVolumeManager(t, eng, liveCatalog(t)) + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{{Name: "data"}}); err != nil { + t.Fatal(err) + } + m.store.recommitting.Store(true) + tmp := m.store.path + ".tmp" + eng.liveVolumeHook = func(verb string) error { + if verb == "detach" { + return os.Mkdir(tmp, 0o700) + } + return nil + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "acme", []types.Volume{}); err == nil { + t.Fatal("expected failed detach result commit") + } + eng.liveVolumeHook = nil + if err := m.DeleteTenant(t.Context(), "acme"); err != nil { + t.Fatal(err) + } + m.mu.Lock() + entry := *m.catalog.entries["data"] + entry.acl = newVolumeACL([]string{"different-tenant"}, volumePublic) + m.catalog.entries["data"] = &entry + m.mu.Unlock() + if err := os.Remove(tmp); err != nil { + t.Fatal(err) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", []types.Volume{{Name: "data"}}); !errors.Is(err, ErrTenantRemoved) { + t.Fatalf("recovery-only retry bypassed tenant removal: %v", err) + } + if got := volumeHoldersOf(m, "data"); got != (volumeHolders{}) { + t.Fatalf("rejected recovery retry reserved a reader: %+v", got) + } + if _, err := m.SetVolumes(t.Context(), sb.ID, "", nil); err != nil { + t.Fatalf("recovery retirement after tenant removal: %v", err) + } + if _, err := m.PatchVolumes(t.Context(), sb.ID, "", types.VolumePatch{Add: []types.Volume{{Name: "other"}}}); !errors.Is(err, ErrTenantRemoved) { + t.Fatalf("repair exemption admitted unrelated addition: %v", err) + } +} + +func TestClaimSnapshotAcknowledgesCapturedRowsBeforeNewerReplacement(t *testing.T) { + s := newClaimStore(t.TempDir(), false) + sb := &types.Sandbox{ID: "owner", Volumes: []types.Volume{{Name: "data"}}} + old := s.set(sb) + entered, resume := make(chan struct{}), make(chan struct{}) + s.replaceFile = func(raw []byte) error { close(entered); <-resume; return s.replace(raw) } + done := make(chan error, 1) + go func() { done <- s.commit(old) }() + <-entered + newer := s.del(sb.ID) + close(resume) + if err := <-done; err != nil { + t.Fatal(err) + } + if !s.pending(newer) { + t.Fatal("captured snapshot acknowledged newer deletion") + } + if refs := s.recoveryVolumes()[sb.ID]; len(refs) != 1 || refs[0].Name != "data" { + t.Fatalf("captured row not acknowledged: %+v", refs) + } + persisted, err := newClaimStore(filepath.Dir(s.path), false).load() + if err != nil || persisted[sb.ID] == nil { + t.Fatalf("captured disk snapshot: %+v, %v", persisted, err) + } + s.replaceFile = nil + if err = s.commit(newer); err != nil { + t.Fatal(err) + } + if len(s.recoveryVolumes()) != 0 { + t.Fatal("empty replacement retained old reference") + } + s.replaceFile = func([]byte) error { t.Fatal("old token attempted another replacement"); return nil } + if err = s.commit(old); err != nil { + t.Fatal(err) + } + persisted, err = newClaimStore(filepath.Dir(s.path), false).load() + if err != nil || len(persisted) != 0 { + t.Fatalf("old token resurrected snapshot: %+v, %v", persisted, err) + } +} diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index a81ff1bb..033bc8c3 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -229,7 +229,7 @@ func TestReserveVolumesExcludesReaderUnderWriter(t *testing.T) { if err := m.reserveVolumes([]types.Volume{{Name: "scratch", Mount: "/volumes/scratch"}}); !errors.Is(err, ErrVolumeBusy) { t.Errorf("read-only reservation under a writer: %v, want ErrVolumeBusy", err) } - if holders := m.volumeAdmission["scratch"]; holders != (volumeHolders{writers: 1}) { + if holders := m.catalog.holders["scratch"]; holders != (volumeHolders{writers: 1}) { t.Errorf("registry after the refusal=%+v, want the writer alone", holders) } } @@ -580,5 +580,5 @@ func newVolumeManagerAt(t *testing.T, eng *fakeEngine, dataDir string, volumes [ func volumeHoldersOf(m *Manager, name string) volumeHolders { m.mu.Lock() defer m.mu.Unlock() - return m.volumeAdmission[name] + return m.catalog.holders[name] } diff --git a/sandboxd/server/api.go b/sandboxd/server/api.go index 6e26e0c9..9a26c5df 100644 --- a/sandboxd/server/api.go +++ b/sandboxd/server/api.go @@ -1,6 +1,8 @@ package server import ( + "errors" + "slices" "time" "github.com/cocoonstack/sandbox/sandboxd/types" @@ -42,6 +44,19 @@ func (r ClaimRequest) Key() types.PoolKey { return types.PoolKey{Template: r.Template, Net: r.Net, Size: r.Size}.Defaulted() } +func (r ClaimRequest) normalizedVolumes() ([]types.Volume, error) { + volumes := slices.Clone(r.Volumes) + if r.VolumesAttachOnly { + if len(volumes) == 0 { + return nil, errors.New("volumes_attach_only requires at least one volume") + } + for i := range volumes { + volumes[i].AttachOnly = true + } + } + return types.ValidateVolumes(volumes) +} + // ClaimResponse is the wire reply of POST /v1/claim; success and Redirect are exclusive. type ClaimResponse struct { ID string `json:"id,omitempty"` diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 78b44fd6..13fba1d5 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -47,6 +47,10 @@ var poolErrHTTP = []struct { }{ {pool.ErrBadKey, http.StatusBadRequest, ""}, {pool.ErrBadVolume, http.StatusBadRequest, ""}, + {pool.ErrVolumeRegistrationConflict, http.StatusConflict, pool.ErrVolumeRegistrationConflict.Error()}, + {pool.ErrVolumeCatalogReadOnly, http.StatusConflict, pool.ErrVolumeCatalogReadOnly.Error()}, + {pool.ErrVolumeCatalogInvalid, http.StatusConflict, pool.ErrVolumeCatalogInvalid.Error()}, + {pool.ErrVolumeCatalogUnavailable, http.StatusServiceUnavailable, "volume catalog unavailable; retry"}, {pool.ErrBadName, http.StatusBadRequest, ""}, {pool.ErrBadCount, http.StatusBadRequest, ""}, {pool.ErrBadEnv, http.StatusBadRequest, ""}, @@ -63,6 +67,9 @@ var poolErrHTTP = []struct { {pool.ErrNoEgressFork, http.StatusConflict, ""}, {pool.ErrVolumeCapture, http.StatusConflict, ""}, {pool.ErrVolumeBusy, http.StatusConflict, ""}, + {pool.ErrReplicaPresent, http.StatusServiceUnavailable, ""}, + {pool.ErrReplicatingPool, http.StatusConflict, ""}, + {pool.ErrReplicaStranded, http.StatusConflict, ""}, {pool.ErrVolumeNeedsRecovery, http.StatusConflict, ""}, {pool.ErrArchived, http.StatusConflict, ""}, {pool.ErrPaused, http.StatusConflict, ""}, @@ -97,6 +104,8 @@ type Manager interface { TenantClaims() map[string]int VolumePlacement(key types.PoolKey, tenant string, names []string) (bool, error) Volumes(tenant string, holders map[string]int) []types.VolumeInfo + RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error + UnregisterVolume(ctx context.Context, name string) error GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) SetVolumes(ctx context.Context, id, tenant string, volumes []types.Volume) (types.SandboxVolumesResponse, error) PatchVolumes(ctx context.Context, id, tenant string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) @@ -269,6 +278,8 @@ func (s *Server) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/claim", s.requireToken(s.handleClaim)) mux.HandleFunc("GET /v1/volumes", s.requireToken(s.handleVolumes)) + mux.HandleFunc("PUT /v1/volumes/{name}", s.requireRoot(s.handleVolumeRegistration)) + mux.HandleFunc("DELETE /v1/volumes/{name}", s.requireRoot(s.handleVolumeRegistration)) mux.HandleFunc("POST /v1/sandboxes/{id}/release", s.handleSandboxVerb("release", s.mgr.Release)) mux.HandleFunc("POST /v1/sandboxes/{id}/hibernate", s.handleSandboxVerb("hibernate", s.mgr.Hibernate)) mux.HandleFunc("POST /v1/sandboxes/{id}/wake", s.handleSandboxVerb("wake", s.mgr.Wake)) @@ -350,7 +361,7 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { } func (s *Server) handleVolumeClaim(w http.ResponseWriter, r *http.Request, req ClaimRequest, key types.PoolKey, hash, tenant string) { - volumes, err := types.ValidateVolumes(req.Volumes, req.VolumesAttachOnly) + volumes, err := req.normalizedVolumes() if err != nil { writeErr(w, http.StatusBadRequest, fmt.Sprintf("%v: %v", pool.ErrBadVolume, err)) return diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index 48bf2a94..1201fb97 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -22,6 +22,19 @@ import ( const retryAfterSeconds = "10" +// Keep HTTP responses readable by SDKs that echo the historical volume shape. +// Storage and forwarded requests retain the canonical AttachOnly field. +type volumeView struct { + Name string `json:"name"` + Mount string `json:"mount,omitempty"` + Mode string `json:"mode,omitempty"` + AttachOnly bool `json:"-"` +} + +var responseMarshalers = json.MarshalToFunc(func(out *jsontext.Encoder, v types.Volume) error { + return json.MarshalEncode(out, volumeView(v)) +}) + type tenantKey struct{} func (s *Server) writeRedirect(w http.ResponseWriter, addrs []string) bool { @@ -212,7 +225,7 @@ func noContent(w http.ResponseWriter) func() { func writeJSON(w http.ResponseWriter, code int, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) - _ = json.MarshalWrite(w, v, jsontext.AllowInvalidUTF8(true)) + _ = json.MarshalWrite(w, v, jsontext.AllowInvalidUTF8(true), json.WithMarshalers(responseMarshalers)) } func writeErr(w http.ResponseWriter, code int, msg string) { diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 6debba66..f24769d1 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2482,6 +2482,8 @@ type checkpointClaimFunc func(string) (*types.Sandbox, error) type idActionFunc func(string) error type fakeManager struct { + registerVolume func(context.Context, string, types.VolumeRegistration) error + unregisterVolume func(context.Context, string) error getVolumes func(string, string) (types.SandboxVolumesResponse, error) setVolumes func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) patchVolumes func(string, string, types.VolumePatch) (types.SandboxVolumesResponse, error) @@ -2557,6 +2559,14 @@ type fakeManager struct { reader *fakeReader } +func (f *fakeManager) RegisterVolume(ctx context.Context, name string, spec types.VolumeRegistration) error { + return f.registerVolume(ctx, name, spec) +} + +func (f *fakeManager) UnregisterVolume(ctx context.Context, name string) error { + return f.unregisterVolume(ctx, name) +} + func (f *fakeManager) GetVolumes(id, tenant string) (types.SandboxVolumesResponse, error) { return f.getVolumes(id, tenant) } diff --git a/sandboxd/server/volume_mutation.go b/sandboxd/server/volume_mutation.go index 1e4f86f9..86dd6398 100644 --- a/sandboxd/server/volume_mutation.go +++ b/sandboxd/server/volume_mutation.go @@ -60,7 +60,7 @@ func (s *Server) handleSandboxVolumes(w http.ResponseWriter, r *http.Request) { code, view.Error = mapped, msg } } - if code >= http.StatusInternalServerError { + if code >= http.StatusInternalServerError && !errors.Is(err, pool.ErrReplicaPresent) { log.WithFunc("server.handleSandboxVolumes").Errorf(r.Context(), err, "converge volumes for %s", id) } } diff --git a/sandboxd/server/volume_mutation_test.go b/sandboxd/server/volume_mutation_test.go index 48ff46b6..2c593b41 100644 --- a/sandboxd/server/volume_mutation_test.go +++ b/sandboxd/server/volume_mutation_test.go @@ -153,7 +153,7 @@ func TestLiveVolumeRoutesHideOtherTenants(t *testing.T) { func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { const cause = "cannot unmount /data; close guest references and retry" - for _, refusal := range []error{pool.ErrVolumeBusy, pool.ErrVolumeCapture} { + for _, refusal := range []error{pool.ErrVolumeBusy, pool.ErrVolumeCapture, pool.ErrReplicaPresent, pool.ErrReplicatingPool, pool.ErrReplicaStranded} { t.Run(refusal.Error(), func(t *testing.T) { mgr := &fakeManager{setVolumes: func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) { return types.SandboxVolumesResponse{VolumesWant: []types.Volume{{Name: "old"}}, States: []types.VolumeState{{Name: "old", State: "pending", Error: cause}}, Error: "volume change failed; retry the desired target"}, fmt.Errorf("%w: volume %q", refusal, "data") @@ -165,9 +165,66 @@ func TestLiveVolumeRoutesReportRefusalsOverRecordedDiagnostics(t *testing.T) { if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatal(err) } - if resp.StatusCode != http.StatusConflict || !strings.HasPrefix(got.Error, refusal.Error()) || len(got.States) != 1 || got.States[0].Error != cause { + wantStatus := http.StatusConflict + if errors.Is(refusal, pool.ErrReplicaPresent) { + wantStatus = http.StatusServiceUnavailable + if resp.Header.Get("Retry-After") == "" { + t.Fatal("temporary replica retirement refusal has no Retry-After") + } + } else if resp.Header.Get("Retry-After") != "" { + t.Fatal("permanent conflict advertises a retry delay") + } + if resp.StatusCode != wantStatus || !strings.HasPrefix(got.Error, refusal.Error()) || len(got.States) != 1 || got.States[0].Error != cause { t.Fatalf("status=%d view=%+v", resp.StatusCode, got) } }) } } + +func TestLiveVolumeRoutesPreserveMixedAttachOnlyRoundtrip(t *testing.T) { + want := []types.Volume{{Name: "mounted", Mount: "/volumes/mounted"}, {Name: "raw", Mode: types.VolumeModeRW, AttachOnly: true}} + legacy := slices.Clone(want) + legacy[1].AttachOnly = false + view := types.SandboxVolumesResponse{VolumesWant: want, Volumes: want} + mgr := &fakeManager{ + getVolumes: func(string, string) (types.SandboxVolumesResponse, error) { return view, nil }, + setVolumes: func(_, _ string, got []types.Volume) (types.SandboxVolumesResponse, error) { + if !slices.Equal(got, legacy) { + t.Errorf("GET-to-PUT changed mixed intent: %+v", got) + } + return view, nil + }, + patchVolumes: func(_, _ string, patch types.VolumePatch) (types.SandboxVolumesResponse, error) { + if !slices.Equal(patch.Add, want) { + t.Errorf("PATCH lost per-entry intent: %+v", patch.Add) + } + return view, nil + }, + } + ts := newTenantTestServer(t, "root", nil, mgr, nil) + get := doReq(t, http.MethodGet, ts.URL+"/v1/sandboxes/sb_1/volumes", "root", "") + defer get.Body.Close() + var got types.SandboxVolumesResponse + if err := json.UnmarshalRead(get.Body, &got); err != nil || !slices.Equal(got.VolumesWant, legacy) { + t.Fatalf("GET broke the published response shape: %+v, %v", got, err) + } + for _, method := range []string{http.MethodPut, http.MethodPatch} { + key := "volumes" + if method == http.MethodPatch { + key = "add" + } + input := got.VolumesWant + if method == http.MethodPatch { + input = want + } + body, err := json.Marshal(map[string][]types.Volume{key: input}) + if err != nil { + t.Fatal(err) + } + resp := doReq(t, method, ts.URL+"/v1/sandboxes/sb_1/volumes", "root", string(body)) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("%s status=%d", method, resp.StatusCode) + } + } +} diff --git a/sandboxd/server/volume_registration.go b/sandboxd/server/volume_registration.go new file mode 100644 index 00000000..39bad6a6 --- /dev/null +++ b/sandboxd/server/volume_registration.go @@ -0,0 +1,44 @@ +package server + +import ( + "context" + "errors" + "net/http" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func (s *Server) handleVolumeRegistration(w http.ResponseWriter, r *http.Request) { + if s.apiToken == "" { + writeErr(w, http.StatusForbidden, "volume registration requires a configured api token") + return + } + name := r.PathValue("name") + if !types.ValidVolumeName(name) { + writeErr(w, http.StatusBadRequest, "invalid volume name") + return + } + var err error + op := "unregister volume" + if r.Method == http.MethodPut { + req, ok := decodeBodyStrict[types.VolumeRegistration](w, r) + if !ok { + return + } + op = "register volume" + err = s.mgr.RegisterVolume(r.Context(), name, req) + } else { + err = s.mgr.UnregisterVolume(r.Context(), name) + } + switch { + case errors.Is(err, pool.ErrVolumeCatalogUnavailable): + writePoolErr(w, err) + case errors.Is(err, context.DeadlineExceeded): + writeErr(w, http.StatusGatewayTimeout, op+" timed out") + case errors.Is(err, context.Canceled): + writeErr(w, http.StatusRequestTimeout, op+" canceled") + default: + writeResult(w, r, op, name, op+" failed", err, noContent(w)) + } +} diff --git a/sandboxd/server/volume_registration_test.go b/sandboxd/server/volume_registration_test.go new file mode 100644 index 00000000..3dc27e4f --- /dev/null +++ b/sandboxd/server/volume_registration_test.go @@ -0,0 +1,276 @@ +package server + +import ( + "bytes" + "context" + "encoding/json/v2" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestVolumeRegistrationRoutesRequireRoot(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name, apiToken, token string + code int + }{ + {"root", "root", "root", http.StatusNoContent}, + {"tenant", "root", "acme-token", http.StatusForbidden}, + {"sandbox", "root", "sandbox-token", http.StatusUnauthorized}, + {"unknown", "root", "unknown", http.StatusUnauthorized}, + {"missing", "root", "", http.StatusUnauthorized}, + {"open without token", "", "", http.StatusForbidden}, + {"open with tenant token", "", "acme-token", http.StatusForbidden}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + called := false + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, + unregisterVolume: func(context.Context, string) error { called = true; return nil }, + } + ts := newTenantTestServer(t, tt.apiToken, []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) + body := `{"path":"/private/data.img"}` + if tt.code != http.StatusNoContent { + body = `{invalid` + } + resp := doReq(t, method, ts.URL+"/v1/volumes/data", tt.token, body) + defer resp.Body.Close() + got, _ := io.ReadAll(resp.Body) + if resp.StatusCode != tt.code || called != (tt.code == http.StatusNoContent) { + t.Fatalf("status=%d called=%v body=%s, want %d", resp.StatusCode, called, got, tt.code) + } + if tt.code == http.StatusNoContent && len(got) != 0 { + t.Fatalf("204 response body = %q", got) + } + }) + } + } +} + +func TestVolumeRegistrationRequiresConfiguredTokenBeforeReadingBody(t *testing.T) { + path := filepath.Join(t.TempDir(), "data.img") + if err := os.WriteFile(path, []byte("existing backing"), 0o600); err != nil { + t.Fatal(err) + } + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, token := range []string{"", "acme-token", "unknown"} { + t.Run(method+"/"+token, func(t *testing.T) { + called := false + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { called = true; return nil }, + unregisterVolume: func(context.Context, string) error { called = true; return nil }, + } + handler := New("", "", mgr, nil, nil, nil, nil, nil).Handler() + body := bytes.NewBufferString(fmt.Sprintf(`{"path":%q,"writable":true}`, path)) + bodyLen := body.Len() + req := httptest.NewRequest(method, "/v1/volumes/data", body) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusForbidden || called || body.Len() != bodyLen { + t.Fatalf("status=%d called=%v unread=%d/%d, want 403 without reading body or calling manager", w.Code, called, body.Len(), bodyLen) + } + if w.Header().Get("Retry-After") != "" { + t.Fatalf("permanent refusal advertises retry: %v", w.Header()) + } + }) + } + } +} + +func TestVolumeRegistrationRoutesForwardBindingAndContext(t *testing.T) { + type requestKey struct{} + const body = `{"path":"/private/images/../data.img","directio":"auto","writable":true,"tenants":["beta","acme"]}` + want := types.VolumeRegistration{Path: "/private/images/../data.img", DirectIO: "auto", Writable: true, Tenants: []string{"beta", "acme"}} + var registered types.VolumeRegistration + var registeredName, removedName string + checkContext := func(ctx context.Context) { + if ctx.Value(requestKey{}) != "request-value" { + t.Error("manager did not receive the request context") + } + } + mgr := &fakeManager{ + registerVolume: func(ctx context.Context, name string, spec types.VolumeRegistration) error { + checkContext(ctx) + registeredName, registered = name, spec + return nil + }, + unregisterVolume: func(ctx context.Context, name string) error { + checkContext(ctx) + removedName = name + return nil + }, + } + handler := New("root", "", mgr, nil, nil, nil, nil, nil).Handler() + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for range 2 { + req := httptest.NewRequest(method, "/v1/volumes/data-v1", strings.NewReader(body)) + req.Header.Set("Authorization", "Bearer root") + req = req.WithContext(context.WithValue(req.Context(), requestKey{}, "request-value")) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != http.StatusNoContent || w.Body.Len() != 0 { + t.Fatalf("%s status=%d body=%s, want empty 204", method, w.Code, w.Body) + } + } + } + if registeredName != "data-v1" || removedName != "data-v1" || !reflect.DeepEqual(registered, want) { + t.Fatalf("registered %q %+v, removed %q, want data-v1 %+v", registeredName, registered, removedName, want) + } +} + +func TestVolumeRegistrationRejectsInvalidBodies(t *testing.T) { + for _, tt := range []struct{ name, body string }{ + {"missing", ""}, + {"malformed", `{"path":`}, + {"array", `[]`}, + {"unknown field", `{"path":"/private/data.img","extra":true}`}, + {"body name", `{"name":"other","path":"/private/data.img"}`}, + {"duplicate field", `{"path":"/private/a.img","path":"/private/b.img"}`}, + {"trailing object", `{"path":"/private/data.img"}{}`}, + {"trailing text", `{"path":"/private/data.img"}oops`}, + {"wrong writable type", `{"path":"/private/data.img","writable":"true"}`}, + {"wrong tenants type", `{"path":"/private/data.img","tenants":"acme"}`}, + {"oversized", `{"path":"/` + strings.Repeat("a", maxBodyBytes) + `"}`}, + } { + t.Run(tt.name, func(t *testing.T) { + called := false + mgr := &fakeManager{registerVolume: func(context.Context, string, types.VolumeRegistration) error { + called = true + return nil + }} + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, http.MethodPut, ts.URL+"/v1/volumes/data", "root", tt.body) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusBadRequest || called { + t.Fatalf("status=%d called=%v body=%s, want 400 without registration", resp.StatusCode, called, body) + } + }) + } +} + +func TestVolumeRegistrationRoutesValidateNames(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name, encoded string + valid bool + }{ + {"data", "data", true}, + {"a_b-0", "a_b-0", true}, + {"abcdefghijklmnopqrst", "abcdefghijklmnopqrst", true}, + {"Data", "Data", false}, + {"1data", "1data", false}, + {"cocoon-data", "cocoon-data", false}, + {"abcdefghijklmnopqrstu", "abcdefghijklmnopqrstu", false}, + {"data/set", "data%2Fset", false}, + {"data set", "data%20set", false}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + var gotName string + mgr := &fakeManager{ + registerVolume: func(_ context.Context, name string, _ types.VolumeRegistration) error { gotName = name; return nil }, + unregisterVolume: func(_ context.Context, name string) error { gotName = name; return nil }, + } + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, method, ts.URL+"/v1/volumes/"+tt.encoded, "root", `{"path":"/private/data.img"}`) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + wantCode, wantName := http.StatusBadRequest, "" + if tt.valid { + wantCode, wantName = http.StatusNoContent, tt.name + } + if resp.StatusCode != wantCode || gotName != wantName { + t.Fatalf("status=%d name=%q body=%s, want %d name=%q", resp.StatusCode, gotName, body, wantCode, wantName) + } + }) + } + } +} + +func TestVolumeRegistrationRoutesMapSafeErrors(t *testing.T) { + for _, method := range []string{http.MethodPut, http.MethodDelete} { + for _, tt := range []struct { + name string + err error + code int + }{ + {"invalid registration", fmt.Errorf("%w: invalid directio", pool.ErrBadVolume), http.StatusBadRequest}, + {"binding conflict", fmt.Errorf("%w: /private/data.img", pool.ErrVolumeRegistrationConflict), http.StatusConflict}, + {"OS root required", pool.ErrVolumeCatalogReadOnly, http.StatusConflict}, + {"invalid catalog destination", fmt.Errorf("%w: /private/runtime-volumes.json", pool.ErrVolumeCatalogInvalid), http.StatusConflict}, + {"catalog unavailable", fmt.Errorf("%w: sync /private/runtime-volumes.json", pool.ErrVolumeCatalogUnavailable), http.StatusServiceUnavailable}, + {"uncertain cancellation", fmt.Errorf("%w: %w", pool.ErrVolumeCatalogUnavailable, context.Canceled), http.StatusServiceUnavailable}, + {"uncertain timeout", fmt.Errorf("%w: %w", pool.ErrVolumeCatalogUnavailable, context.DeadlineExceeded), http.StatusServiceUnavailable}, + {"timeout", fmt.Errorf("register: %w", context.DeadlineExceeded), http.StatusGatewayTimeout}, + {"canceled", fmt.Errorf("register: %w", context.Canceled), http.StatusRequestTimeout}, + {"unexpected failure", errors.New("open /private/data.img: permission denied"), http.StatusInternalServerError}, + } { + t.Run(method+"/"+tt.name, func(t *testing.T) { + mgr := &fakeManager{ + registerVolume: func(context.Context, string, types.VolumeRegistration) error { return tt.err }, + unregisterVolume: func(context.Context, string) error { return tt.err }, + } + ts := newTestServer(t, "root", mgr, nil) + resp := doReq(t, method, ts.URL+"/v1/volumes/data", "root", `{"path":"/private/data.img"}`) + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + var got map[string]string + if err := json.Unmarshal(body, &got); err != nil || len(got) != 1 || got["error"] == "" { + t.Fatalf("error body = %s, decode error = %v", body, err) + } + if resp.StatusCode != tt.code || strings.Contains(string(body), "/private") { + t.Fatalf("status=%d body=%s, want safe %d", resp.StatusCode, body, tt.code) + } + if tt.code == http.StatusServiceUnavailable { + seconds, err := strconv.Atoi(resp.Header.Get("Retry-After")) + if err != nil || seconds < 1 || !strings.Contains(got["error"], "retry") { + t.Fatalf("503 missing retry guidance: headers=%v body=%s", resp.Header, body) + } + } else if resp.Header.Get("Retry-After") != "" { + t.Fatalf("permanent refusal advertises retry: %v", resp.Header) + } + }) + } + } +} + +func TestVolumeRegistrationKeepsCatalogListingPrivate(t *testing.T) { + for _, token := range []string{"root", "acme-token"} { + t.Run(token, func(t *testing.T) { + mgr := &fakeManager{volumeCatalog: func(tenant string, _ map[string]int) []types.VolumeInfo { + wantTenant := "" + if token == "acme-token" { + wantTenant = "acme" + } + if tenant != wantTenant { + t.Errorf("listing tenant = %q, want %q", tenant, wantTenant) + } + return []types.VolumeInfo{{Name: "data", DefaultMount: "/volumes/data", SizeBytes: 42, Available: true, Nodes: 1, Writable: true}} + }} + ts := newTenantTestServer(t, "root", []tenants.TenantSpec{{Name: "acme", Token: "acme-token"}}, mgr, nil) + resp := doReq(t, http.MethodGet, ts.URL+"/v1/volumes", token, "") + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + const want = `{"volumes":[{"name":"data","default_mount":"/volumes/data","size_bytes":42,"available":true,"nodes":1,"writable":true}]}` + if resp.StatusCode != http.StatusOK || string(body) != want { + t.Fatalf("catalog = %d %s, want 200 %s", resp.StatusCode, body, want) + } + }) + } +} diff --git a/sandboxd/server/volume_wire_test.go b/sandboxd/server/volume_wire_test.go new file mode 100644 index 00000000..a1e52304 --- /dev/null +++ b/sandboxd/server/volume_wire_test.go @@ -0,0 +1,127 @@ +package server + +import ( + "context" + "encoding/json/jsontext" + "encoding/json/v2" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +func TestClaimVolumeCompatibilityExpansion(t *testing.T) { + for _, tt := range []struct { + name string + req ClaimRequest + want []types.Volume + err bool + }{ + {"default mount", ClaimRequest{Volumes: []types.Volume{{Name: "data"}}}, []types.Volume{{Name: "data", Mount: "/volumes/data"}}, false}, + {"legacy all devices", ClaimRequest{VolumesAttachOnly: true, Volumes: []types.Volume{{Name: "data"}, {Name: "work", Mode: types.VolumeModeRW}}}, []types.Volume{{Name: "data", AttachOnly: true}, {Name: "work", Mode: types.VolumeModeRW, AttachOnly: true}}, false}, + {"mixed", ClaimRequest{Volumes: []types.Volume{{Name: "data"}, {Name: "work", AttachOnly: true}}}, []types.Volume{{Name: "data", Mount: "/volumes/data"}, {Name: "work", AttachOnly: true}}, false}, + {"mixed reversed", ClaimRequest{Volumes: []types.Volume{{Name: "work", AttachOnly: true}, {Name: "data"}}}, []types.Volume{{Name: "work", AttachOnly: true}, {Name: "data", Mount: "/volumes/data"}}, false}, + {"empty legacy", ClaimRequest{VolumesAttachOnly: true}, nil, true}, + {"legacy mount conflict", ClaimRequest{VolumesAttachOnly: true, Volumes: []types.Volume{{Name: "data", Mount: "/data"}}}, nil, true}, + } { + t.Run(tt.name, func(t *testing.T) { + before := slices.Clone(tt.req.Volumes) + got, err := tt.req.normalizedVolumes() + if (err != nil) != tt.err || !slices.Equal(got, tt.want) { + t.Fatalf("normalized volumes=%+v err=%v, want %+v error=%t", got, err, tt.want, tt.err) + } + if !slices.Equal(tt.req.Volumes, before) { + t.Fatal("legacy expansion mutated caller input") + } + }) + } +} + +func TestVolumeHTTPResponsesPreservePublishedWireShape(t *testing.T) { + volumes := []types.Volume{{Name: "raw", Mode: types.VolumeModeRW, AttachOnly: true}, {Name: "mounted", Mount: "/data"}} + view := types.SandboxVolumesResponse{Volumes: volumes, VolumesWant: volumes} + summary := pool.SandboxSummary{Volumes: volumes, VolumesWant: volumes} + for name, payload := range map[string]any{ + "claim": ClaimResponse{Volumes: volumes}, + "fork": ForkResponse{Children: []ClaimResponse{{Volumes: volumes}}}, + "volumes": view, + "error": types.SandboxVolumesResponse{Volumes: volumes, VolumesWant: volumes, Error: "retry"}, + "summary": summary, + "list": map[string]any{"sandboxes": []*pool.SandboxSummary{&summary}}, + "pointer": &volumes[0], + } { + t.Run(name, func(t *testing.T) { + reply := httptest.NewRecorder() + writeJSON(reply, http.StatusOK, payload) + body := reply.Body.String() + if !jsontext.Value(body).IsValid() || strings.Contains(body, "attach_only") || !strings.Contains(body, `"name":"raw","mode":"rw"`) { + t.Fatalf("published response shape changed: %s", body) + } + }) + } + for _, payload := range []any{ClaimRequest{Volumes: volumes}, &types.Sandbox{Volumes: volumes, VolumesWant: volumes}} { + raw, err := json.Marshal(payload) + if err != nil || !strings.Contains(string(raw), `"attach_only":true`) { + t.Fatalf("internal serialization lost attachment intent: %s, %v", raw, err) + } + } +} + +func TestVolumeHTTPProjectionPreservesInvalidUTF8Handling(t *testing.T) { + reply := httptest.NewRecorder() + writeJSON(reply, http.StatusOK, []types.Volume{{Name: "raw\xff", AttachOnly: true}}) + if got := reply.Body.String(); !jsontext.Value(got).IsValid() || !strings.Contains(got, "raw�") { + t.Fatalf("response no longer replaces invalid UTF8: %q", got) + } +} + +func TestVolumeRoutesKeepLegacyResponseFields(t *testing.T) { + volumes := []types.Volume{{Name: "raw", Mode: types.VolumeModeRW, AttachOnly: true}} + sb := &types.Sandbox{ID: "sb_1", Token: "tok", Volumes: volumes} + view := types.SandboxVolumesResponse{Volumes: volumes, VolumesWant: volumes} + summary := pool.SandboxSummary{ID: sb.ID, Volumes: volumes, VolumesWant: volumes} + mgr := &fakeManager{ + claim: func(context.Context, types.PoolKey, time.Duration) (*types.Sandbox, error) { return sb, nil }, + fork: func(string, string, int, time.Duration) ([]*types.Sandbox, error) { return []*types.Sandbox{sb}, nil }, + claimCheckpoint: func(string) (*types.Sandbox, error) { return sb, nil }, + sandboxByID: func(string) (pool.SandboxSummary, bool) { return summary, true }, + sandboxIndex: func(string, string) []pool.SandboxSummary { return []pool.SandboxSummary{summary} }, + getVolumes: func(string, string) (types.SandboxVolumesResponse, error) { return view, nil }, + setVolumes: func(string, string, []types.Volume) (types.SandboxVolumesResponse, error) { return view, nil }, + patchVolumes: func(string, string, types.VolumePatch) (types.SandboxVolumesResponse, error) { + return view, pool.ErrBadVolume + }, + } + ts := newTestServer(t, "root", mgr, nil) + for _, tt := range []struct { + method, path, body string + status int + }{ + {http.MethodPost, "/v1/claim", `{"template":"rt:24.04"}`, http.StatusOK}, + {http.MethodPost, "/v1/sandboxes/sb_1/fork", `{"token":"tok","count":1}`, http.StatusOK}, + {http.MethodPost, "/v1/checkpoints/ckpt_1/claim", `{}`, http.StatusOK}, + {http.MethodGet, "/v1/sandboxes/sb_1", "", http.StatusOK}, + {http.MethodGet, "/v1/sandboxes", "", http.StatusOK}, + {http.MethodGet, "/v1/sandboxes/sb_1/volumes", "", http.StatusOK}, + {http.MethodPut, "/v1/sandboxes/sb_1/volumes", `{"volumes":[]}`, http.StatusOK}, + {http.MethodPatch, "/v1/sandboxes/sb_1/volumes", `{"remove":["raw"]}`, http.StatusBadRequest}, + } { + t.Run(tt.method+tt.path, func(t *testing.T) { + reply := doReq(t, tt.method, ts.URL+tt.path, "root", tt.body) + defer reply.Body.Close() + var body map[string]jsontext.Value + if err := json.UnmarshalRead(reply.Body, &body); err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(body) + if err != nil || reply.StatusCode != tt.status || strings.Contains(string(raw), "attach_only") || !strings.Contains(string(raw), `"name":"raw","mode":"rw"`) { + t.Fatalf("legacy response changed: status=%d body=%s err=%v", reply.StatusCode, raw, err) + } + }) + } +} diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index ac0cd6b4..1cee1753 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -22,6 +22,14 @@ type VolumeInfo struct { Writable bool `json:"writable,omitzero"` } +// VolumeRegistration binds an existing backing file to a runtime catalog name. +type VolumeRegistration struct { + Path string `json:"path"` + DirectIO string `json:"directio,omitempty"` + Writable bool `json:"writable,omitzero"` + Tenants []string `json:"tenants,omitempty"` +} + // VolumePatch is the wire body of PATCH /v1/sandboxes/{id}/volumes. type VolumePatch struct { Add []Volume `json:"add,omitzero"` diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 4cc8d307..1c537ead 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -752,8 +752,8 @@ type Volume struct { Name string `json:"name"` Mount string `json:"mount,omitempty"` Mode string `json:"mode,omitempty"` - // AttachOnly carries the claim's flag into validation; past it the empty Mount is the signal. - AttachOnly bool `json:"-"` + // AttachOnly leaves mounting and unmounting to the caller. + AttachOnly bool `json:"attach_only,omitzero"` } // RW reports whether the entry asks for write access. @@ -803,19 +803,11 @@ func VolumeRWNames(volumes []Volume) []string { return names } -// VolumesAttachOnly reports whether the caller mounts the devices itself. -func VolumesAttachOnly(volumes []Volume) bool { - return slices.ContainsFunc(volumes, func(v Volume) bool { return v.AttachOnly }) -} - // ValidateVolumes returns detached entries with defaults filled and modes normalized. -func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { +func ValidateVolumes(volumes []Volume) ([]Volume, error) { if len(volumes) > MaxClaimVolumes { return nil, fmt.Errorf("volumes must contain at most %d entries, got %d", MaxClaimVolumes, len(volumes)) } - if attachOnly && len(volumes) == 0 { - return nil, errors.New("volumes_attach_only requires at least one volume") - } applied := make([]Volume, len(volumes)) names := make(map[string]struct{}, len(volumes)) for i, volume := range volumes { @@ -834,7 +826,7 @@ func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { default: return nil, fmt.Errorf("volumes[%d] mode %q must be %s or %s", i, mode, VolumeModeRO, VolumeModeRW) } - if attachOnly { + if volume.AttachOnly { if volume.Mount != "" { return nil, fmt.Errorf("volumes[%d] mount %q is meaningless when the caller mounts the device itself", i, volume.Mount) } @@ -847,6 +839,9 @@ func ValidateVolumes(volumes []Volume, attachOnly bool) ([]Volume, error) { } for j := range i { other := applied[j].Mount + if other == "" { + continue + } switch { case mount == other: return nil, fmt.Errorf("volumes[%d] mount %q duplicates volumes[%d]", i, mount, j) diff --git a/sandboxd/types/volume_test.go b/sandboxd/types/volume_test.go index 9549af49..c7f428f1 100644 --- a/sandboxd/types/volume_test.go +++ b/sandboxd/types/volume_test.go @@ -31,7 +31,7 @@ func TestValidVolumeName(t *testing.T) { func TestValidateVolumes(t *testing.T) { volumes := []Volume{{Name: "dataset"}, {Name: "weights-1", Mount: "/models"}} wantInput := slices.Clone(volumes) - got, err := ValidateVolumes(volumes, false) + got, err := ValidateVolumes(volumes) if err != nil { t.Fatalf("ValidateVolumes: %v", err) } @@ -54,7 +54,7 @@ func TestValidateVolumes(t *testing.T) { {Name: "f"}, {Name: "g"}, {Name: "h"}, - }, false); err != nil || len(got) != MaxClaimVolumes { + }); err != nil || len(got) != MaxClaimVolumes { t.Errorf("ValidateVolumes at limit: got=%v err=%v", got, err) } @@ -75,7 +75,7 @@ func TestValidateVolumes(t *testing.T) { {"uppercase-mode", []Volume{{Name: "dataset", Mode: "RW"}}}, } { t.Run(tt.name, func(t *testing.T) { - if _, err := ValidateVolumes(tt.volumes, false); err == nil { + if _, err := ValidateVolumes(tt.volumes); err == nil { t.Fatal("ValidateVolumes succeeded") } }) @@ -87,7 +87,7 @@ func TestValidateVolumesNormalizesMode(t *testing.T) { {Name: "shared"}, {Name: "explicit", Mode: VolumeModeRO}, {Name: "writable", Mode: VolumeModeRW}, - }, false) + }) if err != nil { t.Fatalf("ValidateVolumes: %v", err) } @@ -112,10 +112,10 @@ func TestValidateVolumesNormalizesMode(t *testing.T) { func TestValidateVolumesAttachOnly(t *testing.T) { got, err := ValidateVolumes([]Volume{ - {Name: "dataset"}, - {Name: "scratch", Mode: VolumeModeRW}, - {Name: "explicit", Mode: VolumeModeRO}, - }, true) + {Name: "dataset", AttachOnly: true}, + {Name: "scratch", Mode: VolumeModeRW, AttachOnly: true}, + {Name: "explicit", Mode: VolumeModeRO, AttachOnly: true}, + }) if err != nil { t.Fatalf("ValidateVolumes: %v", err) } @@ -127,18 +127,11 @@ func TestValidateVolumesAttachOnly(t *testing.T) { if !slices.Equal(got, want) { t.Errorf("volumes %v, want %v", got, want) } - if !VolumesAttachOnly(got) { - t.Error("validated attach-only set does not report itself") - } - if VolumesAttachOnly(nil) || VolumesAttachOnly([]Volume{{Name: "dataset", Mount: "/volumes/dataset"}}) { - t.Error("mounted set reports itself attach-only") - } for _, tt := range []struct { name string volumes []Volume }{ - {"no volumes", nil}, {"default mount spelled out", []Volume{{Name: "dataset", Mount: "/volumes/dataset"}}}, {"custom mount", []Volume{{Name: "dataset", Mount: "/datasets"}}}, {"one entry of many", []Volume{{Name: "dataset"}, {Name: "scratch", Mount: "/scratch"}}}, @@ -147,7 +140,10 @@ func TestValidateVolumesAttachOnly(t *testing.T) { {"unknown mode", []Volume{{Name: "dataset", Mode: "readwrite"}}}, } { t.Run(tt.name, func(t *testing.T) { - if _, err := ValidateVolumes(tt.volumes, true); err == nil { + for i := range tt.volumes { + tt.volumes[i].AttachOnly = true + } + if _, err := ValidateVolumes(tt.volumes); err == nil { t.Fatal("ValidateVolumes succeeded") } }) @@ -157,19 +153,38 @@ func TestValidateVolumesAttachOnly(t *testing.T) { func TestValidateVolumesRejectsGuestOSMounts(t *testing.T) { for _, root := range append([]string{"/"}, guestOSMountRoots...) { t.Run(root, func(t *testing.T) { - if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root}}, false); err == nil { + if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root}}); err == nil { t.Fatalf("accepted OS mount %q", root) } if root != "/" { - if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root + "/child"}}, false); err == nil { + if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: root + "/child"}}); err == nil { t.Fatalf("accepted mount under OS root %q", root) } } }) } for _, mount := range []string{"/data", "/home/dataset", "/opt-data", "/usrdata"} { - if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: mount}}, false); err != nil { + if _, err := ValidateVolumes([]Volume{{Name: "dataset", Mount: mount}}); err != nil { t.Errorf("rejected allowed mount %q: %v", mount, err) } } } + +func TestValidateMixedVolumesPreservesPerEntryIntent(t *testing.T) { + request := []Volume{{Name: "mounted"}, {Name: "writer", Mode: VolumeModeRW}, {Name: "raw", AttachOnly: true}, {Name: "raw-writer", Mode: VolumeModeRW, AttachOnly: true}} + got, err := ValidateVolumes(request) + if err != nil { + t.Fatal(err) + } + want := []Volume{{Name: "mounted", Mount: "/volumes/mounted"}, {Name: "writer", Mount: "/volumes/writer", Mode: VolumeModeRW}, {Name: "raw", AttachOnly: true}, {Name: "raw-writer", Mode: VolumeModeRW, AttachOnly: true}} + if !slices.Equal(got, want) { + t.Fatalf("mixed intent changed: got %+v, want %+v", got, want) + } + again, err := ValidateVolumes(got) + if err != nil || !slices.Equal(again, want) { + t.Fatalf("second validation changed mixed intent: %+v, %v", again, err) + } + if _, err = ValidateVolumes([]Volume{{Name: "raw", AttachOnly: true, Mount: "/data"}}); err == nil { + t.Fatal("accepted attach-only with a mount") + } +} diff --git a/sdk/go/client.go b/sdk/go/client.go index ccb932fd..b94eb753 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -110,14 +110,31 @@ func (r claimRequest) rejectPinnedAxes() error { return nil } -func (r claimRequest) validateVolumes() error { +func (r *claimRequest) prepareVolumes() error { + attachOnly := 0 for _, v := range r.Volumes { - if v.Mode != "" && v.Mode != volumeModeRW { - return fmt.Errorf("volume %q: mode must be \"\", %q, or %q, got %q", v.Name, volumeModeRO, volumeModeRW, v.Mode) + if err := validateVolume(v); err != nil { + return err } if r.VolumesAttachOnly && v.Mount != "" { return fmt.Errorf("volume %q: mount %q is meaningless with WithVolumesAttachOnly, which leaves mounting to the caller", v.Name, v.Mount) } + if v.AttachOnly { + attachOnly++ + } + } + if !r.VolumesAttachOnly && attachOnly > 0 && attachOnly != len(r.Volumes) { + // Remove this guard only when every supported claim and redirect target honors per-entry attach_only. + return errors.New("mixed mounted and attach-only initial claims are not safe on older servers: claim without volumes, then use SetVolumes or PatchVolumes") + } + if attachOnly > 0 { + r.VolumesAttachOnly = true + } + if r.VolumesAttachOnly { + r.Volumes = slices.Clone(r.Volumes) + for i := range r.Volumes { + r.Volumes[i].AttachOnly = false + } } return nil } @@ -172,7 +189,7 @@ func (c *Client) New(ctx context.Context, template string, opts ...Option) (*San for _, opt := range opts { opt(&claim) } - if err := claim.validateVolumes(); err != nil { + if err := claim.prepareVolumes(); err != nil { return nil, err } addr, cr, err := claimFollow(c.addr, "claim", func(noRedirect, requirePromoted bool) ([]byte, error) { @@ -250,8 +267,9 @@ func (c *Client) ownerAt(ctx context.Context, addr, id, token string) (string, e } func (c *Client) handleFrom(dialed string, cr claimResponse) *Sandbox { + volumes := slices.Clone(cr.Volumes) return &Sandbox{ - ID: cr.ID, Deadline: cr.Deadline, Volumes: cr.Volumes, VolumesWant: slices.Clone(cr.Volumes), + ID: cr.ID, Deadline: cr.Deadline, Volumes: volumes, VolumesWant: slices.Clone(volumes), FromCheckpoint: cr.FromCheckpoint, TemplateDigest: cr.TemplateDigest, NetRoute: cr.NetRoute, c: c, token: cr.Token, owner: cmp.Or(cr.OwnerAddr, dialed), } @@ -312,7 +330,7 @@ func (c *Client) doJSON[T any](ctx context.Context, method, addr, path string, b if resp.StatusCode != http.StatusOK { return out, apiError(verb, resp) } - if err := json.UnmarshalRead(resp.Body, &out); err != nil { + if err := json.UnmarshalRead(resp.Body, &out, json.WithUnmarshalers(responseUnmarshalers)); err != nil { return out, fmt.Errorf("decode %s response: %w", verb, err) } return out, nil diff --git a/sdk/go/options.go b/sdk/go/options.go index 20b75401..277831ad 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -29,12 +29,13 @@ type NetShape string // node's warm pools. type Size string -// Volume requests one catalog entry at an optional guest mount path and mode. +// Volume requests a catalog entry. AttachOnly leaves mounting and unmounting to the caller. type Volume struct { Name string `json:"name"` Mount string `json:"mount,omitempty"` // Mode is "rw" for a writable mount; empty (or "ro") means read-only. - Mode string `json:"mode,omitempty"` + Mode string `json:"mode,omitempty"` + AttachOnly bool `json:"attach_only,omitzero"` } // VolumeInfo describes one visible fleet catalog entry. diff --git a/sdk/go/template.go b/sdk/go/template.go index 34ee31f7..e9e08a9a 100644 --- a/sdk/go/template.go +++ b/sdk/go/template.go @@ -29,7 +29,7 @@ func (t *Template) New(ctx context.Context, opts ...Option) (*Sandbox, error) { if err := claim.rejectPinnedAxes(); err != nil { return nil, err } - if err := claim.validateVolumes(); err != nil { + if err := claim.prepareVolumes(); err != nil { return nil, err } claim.Net, claim.Size = t.net, t.size diff --git a/sdk/go/volume_mutation_test.go b/sdk/go/volume_mutation_test.go index 7f65427c..3c258326 100644 --- a/sdk/go/volume_mutation_test.go +++ b/sdk/go/volume_mutation_test.go @@ -51,7 +51,7 @@ func TestLiveVolumesUseOwnerAndAPIToken(t *testing.T) { func TestLiveVolumeFailureRetainsPartialView(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) - _, _ = io.WriteString(w, `{"volumes_want":[{"name":"new","mount":"/new"}],"volumes":[{"name":"old","mount":"/old"}],"converging":false,"states":[{"name":"new","state":"pending","error":"retry"},{"name":"old","state":"detaching"}],"error":"convergence failed"}`) + _, _ = io.WriteString(w, `{"volumes_want":[{"name":"new","mode":"rw"}],"volumes":[{"name":"old"}],"converging":false,"states":[{"name":"new","state":"pending","error":"retry"},{"name":"old","state":"detaching"}],"error":"convergence failed"}`) })) t.Cleanup(ts.Close) c := testClient(t, ts) @@ -63,6 +63,11 @@ func TestLiveVolumeFailureRetainsPartialView(t *testing.T) { if !isVolume || !isAPI || api.Status != http.StatusInternalServerError || failure.Response == nil { t.Fatalf("%s error lost response or APIError: %v", method, err) } + for _, entries := range [][]Volume{got.Volumes, got.VolumesWant, sb.Volumes, sb.VolumesWant} { + if len(entries) != 1 || !entries[0].AttachOnly { + t.Fatalf("%s partial view lost attach_only: %+v", method, entries) + } + } if !reflect.DeepEqual(got, *failure.Response) || got.States[0].State != "pending" || got.States[1].State != "detaching" || sb.VolumesWant[0].Name != "new" || sb.Volumes[0].Name != "old" { t.Fatalf("partial view=%+v handle=%+v", got, sb) } @@ -188,3 +193,114 @@ func callVolumes(t *testing.T, sb *Sandbox, method string) (SandboxVolumesRespon return sb.GetVolumes(t.Context()) } } + +func TestLiveVolumeAttachOnlyRoundTrip(t *testing.T) { + const entries = `[{"name":"mounted-ro","mount":"/volumes/mounted-ro"},{"name":"mounted-rw","mount":"/work","mode":"rw"},{"name":"device-ro","attach_only":true},{"name":"device-rw","mode":"rw","attach_only":true}]` + var forwarded []map[string]any + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut { + var body struct { + Volumes []map[string]any `json:"volumes"` + } + if err := json.UnmarshalRead(r.Body, &body); err != nil { + t.Errorf("decode replacement: %v", err) + } + forwarded = body.Volumes + } + _, _ = io.WriteString(w, `{"volumes_want":`+entries+`,"volumes":`+entries+`,"states":[],"converging":false}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + sb := &Sandbox{ID: "sb_1", owner: c.addr, c: c} + view, err := sb.GetVolumes(t.Context()) + if err != nil { + t.Fatal(err) + } + if _, err = sb.SetVolumes(t.Context(), view.VolumesWant); err != nil { + t.Fatal(err) + } + var want []map[string]any + if err = json.Unmarshal([]byte(entries), &want); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(forwarded, want) { + t.Fatalf("GET-to-PUT changed volume intent: got=%v want=%v", forwarded, want) + } + if !slices.Equal(sb.Volumes, view.Volumes) || !slices.Equal(sb.VolumesWant, view.VolumesWant) { + t.Fatal("roundtrip changed handle state") + } +} + +func TestLiveVolumesRejectConflictingAttachOnlyBeforeRequest(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("invalid volume reached server") })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + sb := c.Attach(c.addr, "sb_1", "tok") + volumes := []Volume{{Name: "data", Mount: "/data", AttachOnly: true}} + if _, err := sb.SetVolumes(t.Context(), volumes); err == nil { + t.Fatal("SetVolumes accepted mount with attach_only") + } + if _, err := sb.PatchVolumes(t.Context(), VolumePatch{Add: volumes}); err == nil { + t.Fatal("PatchVolumes accepted mount with attach_only") + } +} + +func TestVolumeResponseInferenceDoesNotChangeRequestDecoding(t *testing.T) { + var request Volume + if err := json.Unmarshal([]byte(`{"name":"default-mounted"}`), &request); err != nil || request.AttachOnly { + t.Fatalf("ordinary request decode acquired raw intent: %+v err=%v", request, err) + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/sandboxes" { + _, _ = io.WriteString(w, `{"sandboxes":[{"id":"sb_1","volumes":[{"name":"raw"}],"volumes_want":[{"name":"mounted","mount":"/data"}]},{"id":"sb_2","volumes":[],"volumes_want":[]},{"id":"sb_3"}]}`) + return + } + _, _ = io.WriteString(w, `{"volumes":[{"name":"raw"}],"volumes_want":[{"name":"raw","mount":""}],"states":[],"converging":false}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + rows, err := c.Sandboxes(t.Context()) + if err != nil || !rows[0].Volumes[0].AttachOnly || rows[0].VolumesWant[0].AttachOnly || rows[1].Volumes == nil || rows[2].Volumes != nil { + t.Fatalf("summary inference or nil/empty changed: %+v err=%v", rows, err) + } + sb := c.Attach(c.addr, "sb_1", "tok") + view, err := sb.GetVolumes(t.Context()) + if err != nil || !view.Volumes[0].AttachOnly || !view.VolumesWant[0].AttachOnly { + t.Fatalf("live inference: %+v err=%v", view, err) + } +} + +func TestClaimResponseDecoderAndOwnedCopies(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"id":"sb_1","token":"tok","volumes":[{"name":"raw"},{"name":"mounted","mount":"/data"}]}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + wire, err := c.doJSON[claimResponse](t.Context(), http.MethodPost, c.addr, "/v1/claim", nil, "", "claim") + if err != nil { + t.Fatal(err) + } + sb := c.handleFrom("node", wire) + if !sb.Volumes[0].AttachOnly || sb.Volumes[1].AttachOnly || !wire.Volumes[0].AttachOnly { + t.Fatalf("response decode lost attach-only state: handle=%+v wire=%+v", sb.Volumes, wire) + } + sb.Volumes[0].Name = "caller" + if wire.Volumes[0].Name != "raw" || sb.VolumesWant[0].Name != "raw" { + t.Fatal("claim volume views alias the wire or each other") + } + if c.handleFrom("node", claimResponse{}).Volumes != nil || c.handleFrom("node", claimResponse{Volumes: []Volume{}}).Volumes == nil { + t.Fatal("claim response changed nil versus empty volume semantics") + } +} + +func TestForkResponseDecodesNestedVolumes(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"children":[{"id":"sb_child","token":"tok","volumes":[{"name":"raw"},{"name":"mounted","mount":"/data"}]}]}`) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + reply, err := c.doJSON[forkResponse](t.Context(), http.MethodPost, c.addr, "/v1/sandboxes/sb_1/fork", nil, "", "fork") + if err != nil || len(reply.Children) != 1 || !reply.Children[0].Volumes[0].AttachOnly || reply.Children[0].Volumes[1].AttachOnly { + t.Fatalf("nested fork volumes did not decode: %+v err=%v", reply, err) + } +} diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go index 69e1dea4..a632cc70 100644 --- a/sdk/go/volumes.go +++ b/sdk/go/volumes.go @@ -3,6 +3,7 @@ package sandbox import ( "bytes" "context" + "encoding/json/jsontext" "encoding/json/v2" "errors" "fmt" @@ -55,21 +56,54 @@ func (s *Sandbox) GetVolumes(ctx context.Context) (SandboxVolumesResponse, error return s.requestVolumes(ctx, http.MethodGet, nil, "get volumes") } -// SetVolumes converges the full desired set; an explicit empty slice clears mutable mounts, and nil is rejected. +// SetVolumes converges the full desired set; an explicit empty slice clears volumes, and nil is rejected. func (s *Sandbox) SetVolumes(ctx context.Context, volumes []Volume) (SandboxVolumesResponse, error) { if volumes == nil { return SandboxVolumesResponse{}, errors.New("set volumes: use an explicit empty slice to clear volumes") } + for _, volume := range volumes { + if err := validateVolume(volume); err != nil { + return SandboxVolumesResponse{}, err + } + } return s.requestVolumes(ctx, http.MethodPut, struct { Volumes []Volume `json:"volumes"` }{Volumes: volumes}, "set volumes") } -// PatchVolumes adds or removes read-only mounted entries by name, retaining immutable entries. +// PatchVolumes adds or removes catalog volumes by name. func (s *Sandbox) PatchVolumes(ctx context.Context, patch VolumePatch) (SandboxVolumesResponse, error) { + for _, volume := range patch.Add { + if err := validateVolume(volume); err != nil { + return SandboxVolumesResponse{}, err + } + } return s.requestVolumes(ctx, http.MethodPatch, patch, "patch volumes") } +func validateVolume(v Volume) error { + if v.Mode != "" && v.Mode != volumeModeRO && v.Mode != volumeModeRW { + return fmt.Errorf("volume %q: mode must be \"\", %q, or %q, got %q", v.Name, volumeModeRO, volumeModeRW, v.Mode) + } + if v.AttachOnly && v.Mount != "" { + return fmt.Errorf("volume %q: mount %q is meaningless with attach_only, which leaves mounting to the caller", v.Name, v.Mount) + } + return nil +} + +// Historical responses identify attach-only devices by their absent mount. +type responseVolume Volume + +var responseUnmarshalers = json.UnmarshalFromFunc(func(in *jsontext.Decoder, v *Volume) error { + var wire responseVolume + if err := json.UnmarshalDecode(in, &wire); err != nil { + return err + } + *v = Volume(wire) + v.AttachOnly = v.Mount == "" + return nil +}) + func (s *Sandbox) requestVolumes(ctx context.Context, method string, request any, verb string) (SandboxVolumesResponse, error) { var out SandboxVolumesResponse var body io.Reader @@ -92,7 +126,7 @@ func (s *Sandbox) requestVolumes(ctx context.Context, method string, request any Converging *bool `json:"converging"` Error string `json:"error"` } - decodeErr := json.UnmarshalRead(resp.Body, &wire) + decodeErr := json.UnmarshalRead(resp.Body, &wire, json.WithUnmarshalers(responseUnmarshalers)) complete := decodeErr == nil && wire.VolumesWant != nil && wire.Volumes != nil && wire.States != nil && wire.Converging != nil if complete { out = SandboxVolumesResponse{VolumesWant: *wire.VolumesWant, Volumes: *wire.Volumes, States: *wire.States, Converging: *wire.Converging, Error: wire.Error} diff --git a/sdk/go/volumes_test.go b/sdk/go/volumes_test.go index d8a2c14e..b4a4da9d 100644 --- a/sdk/go/volumes_test.go +++ b/sdk/go/volumes_test.go @@ -3,6 +3,7 @@ package sandbox import ( "context" "encoding/json/v2" + "fmt" "net/http" "net/http/httptest" "slices" @@ -253,7 +254,11 @@ func TestWithVolumesAttachOnlySendsFlagAndDecodesMountlessEcho(t *testing.T) { if mount, present := entry["mount"]; present { t.Errorf("volume %v carries mount %v, want none", entry["name"], mount) } + if _, present := entry["attach_only"]; present { + t.Error("legacy attach-only claim carries a per-entry flag") + } } + echo[0].AttachOnly, echo[1].AttachOnly = true, true if !slices.Equal(sb.Volumes, echo) { t.Errorf("volumes = %+v, want %+v", sb.Volumes, echo) } @@ -301,7 +306,7 @@ func TestRejectsInvalidVolumeModeLocally(t *testing.T) { } func TestSandboxVolumesEchoMode(t *testing.T) { - want := []Volume{{Name: "imagenet"}, {Name: "scratch", Mode: "rw"}} + want := []Volume{{Name: "imagenet", Mount: "/volumes/imagenet"}, {Name: "scratch", Mount: "/volumes/scratch", Mode: "rw"}} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, claimResponse{ID: "sb_1", Token: "tok", Volumes: want}) })) @@ -315,3 +320,76 @@ func TestSandboxVolumesEchoMode(t *testing.T) { t.Errorf("volumes = %+v, want %+v", sb.Volumes, want) } } + +func TestPerEntryAttachOnlyClaimCompatibility(t *testing.T) { + for _, promoted := range []bool{false, true} { + for _, mixed := range []bool{false, true} { + t.Run(fmt.Sprintf("promoted=%t/mixed=%t", promoted, mixed), func(t *testing.T) { + volumes := []Volume{{Name: "device-ro", AttachOnly: true}, {Name: "device-rw", Mode: "rw", AttachOnly: true}} + if mixed { + volumes = append(volumes, Volume{Name: "mounted-ro"}) + } + var got claimRequest + requests := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if err := json.UnmarshalRead(r.Body, &got); err != nil { + t.Errorf("decode claim: %v", err) + } + // An older server understands only the claim-wide flag. + _ = json.MarshalWrite(w, claimResponse{ID: "sb_1", Token: "tok", Volumes: got.Volumes}) + })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + option := WithVolumes(volumes...) + var sb *Sandbox + var err error + if promoted { + tpl := &Template{Name: "task:v1", c: c, addr: c.addr, net: "none", size: "small"} + sb, err = tpl.New(t.Context(), option) + } else { + sb, err = c.New(t.Context(), "rt:24.04", option) + } + if mixed { + if err == nil || !strings.Contains(err.Error(), "claim without volumes") || requests != 0 { + t.Fatalf("mixed claim reached server: requests=%d err=%v", requests, err) + } + return + } + if err != nil { + t.Fatal(err) + } + if !got.VolumesAttachOnly || !slices.Equal(sb.Volumes, volumes) { + t.Fatalf("claim=%+v reply=%+v want=%+v", got, sb.Volumes, volumes) + } + for _, v := range got.Volumes { + if v.AttachOnly { + t.Fatal("claim retained per-entry attach_only") + } + } + if _, err = c.New(t.Context(), "rt:24.04", option); err != nil || !got.VolumesAttachOnly { + t.Fatalf("reused option lost raw intent: claim=%+v err=%v", got, err) + } + }) + } + } +} + +func TestPerEntryAttachOnlyRejectsClaimMount(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("invalid claim reached server") })) + t.Cleanup(ts.Close) + c := testClient(t, ts) + for _, promoted := range []bool{false, true} { + var err error + option := WithVolumes(Volume{Name: "data", Mount: "/data", AttachOnly: true}) + if promoted { + tpl := &Template{Name: "task:v1", c: c, addr: c.addr} + _, err = tpl.New(t.Context(), option) + } else { + _, err = c.New(t.Context(), "rt:24.04", option) + } + if err == nil || !strings.Contains(err.Error(), "meaningless with attach_only") { + t.Fatalf("conflicting mount: %v", err) + } + } +} diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index aa3c8fd1..843ca135 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -18,6 +18,7 @@ from .errors import APIError, SandboxTimeout, require_field from .sandbox import Sandbox from .template import _claim_body, _template_query +from .volumes import Volume, _response_volumes, _sandbox_response T = TypeVar("T") @@ -60,7 +61,7 @@ def new( size: str = "", ttl_seconds: int = 0, claim_ref: str = "", - volumes: list[str | Mapping[str, str]] | None = None, + volumes: Iterable[str | Mapping[str, str | bool] | Volume] | None = None, mount: bool = True, *, deadline: float | None = None, @@ -109,7 +110,7 @@ def attach(self, owner_addr: str, id: str, token: str) -> Sandbox: def sandboxes(self) -> list[dict[str, Any]]: """Lists the claims this token may see: id, key, deadline, claim_ref — never tokens or host paths.""" reply = self._request(self.addr, "GET", "/v1/sandboxes", None, "list sandboxes") - return [dict(sb) for sb in reply.get("sandboxes") or []] + return [_sandbox_response(sb) for sb in reply.get("sandboxes") or []] def drain(self) -> dict[str, Any]: """Cordons the node (root token): new claims are refused, live ones run to their leases.""" @@ -180,7 +181,7 @@ def _handle_from(self, dialed: str, reply: dict[str, Any], verb: str = "claim") deadline=reply.get("deadline", ""), from_checkpoint=reply.get("from_checkpoint", ""), template_digest=reply.get("template_digest", ""), - volumes=reply.get("volumes") or [], + volumes=_response_volumes(reply.get("volumes") or []), ) def _post_json( @@ -217,11 +218,17 @@ def _request( reply = json.loads(raw) except (ValueError, TypeError): reply = None - raise APIError(verb, status, _error_message(raw), reply=reply if isinstance(reply, dict) else None) + raise APIError( + verb, + status, + _error_message(raw), + reply=reply if isinstance(reply, dict) else None, + ) if not raw: return {} try: - return cast(dict[str, Any], json.loads(raw)) + reply = json.loads(raw) + return cast(dict[str, Any], reply) except json.JSONDecodeError as exc: raise APIError(verb, 0, "malformed JSON in response") from exc diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 1a83fa86..4981a222 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -7,7 +7,7 @@ import threading import time from collections import deque -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, cast from .checkpoint import Checkpoint @@ -24,7 +24,7 @@ ) from .frames import BULK_CHUNK, FS_CHUNK, KEEP_ALIVE_PROTO from .template import Template -from .volumes import SandboxVolumesResponse, Volume, _volume_body, _volume_response +from .volumes import SandboxVolumesResponse, Volume, _volume_bodies, _volume_response if TYPE_CHECKING: from .client import Client @@ -45,7 +45,7 @@ def __init__( deadline: str = "", from_checkpoint: str = "", template_digest: str = "", - volumes: list[dict[str, str]] | None = None, + volumes: Sequence[Mapping[str, str | bool] | Volume] | None = None, ) -> None: self._client = client self.id = id @@ -54,7 +54,7 @@ def __init__( self.deadline = deadline self.from_checkpoint = from_checkpoint self.template_digest = template_digest - self.volumes = [dict(volume) for volume in volumes or []] + self.volumes = [dict(cast("Mapping[str, str | bool]", volume)) for volume in volumes or []] self.volumes_want = [dict(volume) for volume in self.volumes] self._pool = ConnPool(client.keep_alive) self._proto = 0 @@ -74,28 +74,26 @@ def get_volumes(self, *, timeout: float | None = None) -> SandboxVolumesResponse return self._volumes_request("GET", None, "get volumes", timeout) def set_volumes( - self, volumes: list[str | Mapping[str, str] | Volume], *, timeout: float | None = None + self, volumes: Iterable[str | Mapping[str, str | bool] | Volume], *, timeout: float | None = None ) -> SandboxVolumesResponse: - """Replaces the desired set and converges; an explicit empty list clears mutable mounts.""" - if volumes is None or isinstance(volumes, str): + """Replaces the desired set and converges; an explicit empty list clears volumes.""" + if volumes is None: raise TypeError("set volumes requires an explicit list; use [] to clear volumes") - return self._volumes_request( - "PUT", {"volumes": [_volume_body(volume, True) for volume in volumes]}, "set volumes", timeout - ) + return self._volumes_request("PUT", {"volumes": _volume_bodies(volumes, True)}, "set volumes", timeout) def patch_volumes( self, *, - add: list[str | Mapping[str, str] | Volume] | None = None, + add: Iterable[str | Mapping[str, str | bool] | Volume] | None = None, remove: list[str] | None = None, timeout: float | None = None, ) -> SandboxVolumesResponse: - """Adds or removes mounted read-only entries by name; omitted arguments leave the set unchanged.""" - if isinstance(add, str) or isinstance(remove, str): + """Adds or removes catalog volumes by name; omitted arguments leave the set unchanged.""" + if isinstance(remove, (str, bytes, bytearray)): raise TypeError("patch volumes takes lists of volumes and names") body: dict[str, Any] = {} if add is not None: - body["add"] = [_volume_body(volume, True) for volume in add] + body["add"] = _volume_bodies(add, True) if remove is not None: body["remove"] = list(remove) return self._volumes_request("PATCH", body, "patch volumes", timeout) @@ -445,8 +443,8 @@ def _volumes_request( return view def _record_volumes(self, view: SandboxVolumesResponse) -> None: - self.volumes_want = [cast(dict[str, str], volume.copy()) for volume in view["volumes_want"]] - self.volumes = [cast(dict[str, str], volume.copy()) for volume in view["volumes"]] + self.volumes_want = [cast("dict[str, str | bool]", volume.copy()) for volume in view["volumes_want"]] + self.volumes = [cast("dict[str, str | bool]", volume.copy()) for volume in view["volumes"]] def _dial(self, deadline: float | None = None) -> Conn: return self._client._dial(self.owner, self.id, self.token, deadline) diff --git a/sdk/python/cocoonsandbox/template.py b/sdk/python/cocoonsandbox/template.py index 5e468aa0..d477679a 100644 --- a/sdk/python/cocoonsandbox/template.py +++ b/sdk/python/cocoonsandbox/template.py @@ -3,10 +3,10 @@ from __future__ import annotations import urllib.parse -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any -from .volumes import _volume_body +from .volumes import Volume, _volume_bodies if TYPE_CHECKING: from .client import Client @@ -27,7 +27,7 @@ def __init__(self, client: Client, addr: str, name: str, net: str, size: str, co def new( self, ttl_seconds: int = 0, - volumes: list[str | Mapping[str, str]] | None = None, + volumes: Iterable[str | Mapping[str, str | bool] | Volume] | None = None, mount: bool = True, *, deadline: float | None = None, @@ -36,7 +36,7 @@ def new( mount=False attaches the volumes without mounting them.""" claim = _claim_body(self.name, self.net, self.size, ttl_seconds, volumes, mount) claim["require_promoted"] = True - if volumes: + if claim.get("volumes"): return self._client._claim_from(self._addr, claim, deadline=deadline) claim["no_redirect"] = True reply = self._client._post_json(self._addr, "/v1/claim", claim, "claim", deadline=deadline) @@ -55,7 +55,7 @@ def _claim_body( net: str, size: str, ttl_seconds: int, - volumes: list[str | Mapping[str, str]] | None = None, + volumes: Iterable[str | Mapping[str, str | bool] | Volume] | None = None, mount: bool = True, claim_ref: str = "", ) -> dict[str, Any]: @@ -66,8 +66,22 @@ def _claim_body( claim["size"] = size if ttl_seconds: claim["ttl_seconds"] = ttl_seconds - if volumes: - claim["volumes"] = [_volume_body(volume, mount) for volume in volumes] + if volumes is not None: + entries = _volume_bodies(volumes, mount) + raw = sum(bool(entry.get("attach_only")) for entry in entries) + if mount and raw and raw != len(entries): + # Remove this guard only when every supported claim and redirect target honors per-entry attach_only. + raise TypeError( + "mixed mounted and attach-only initial claims are not safe on older servers: " + "claim without volumes, then use set_volumes or patch_volumes" + ) + if raw: + mount = False + if not mount: + for entry in entries: + entry.pop("attach_only", None) + if entries: + claim["volumes"] = entries if not mount: claim["volumes_attach_only"] = True if claim_ref: diff --git a/sdk/python/cocoonsandbox/volumes.py b/sdk/python/cocoonsandbox/volumes.py index ce156554..7cca6cac 100644 --- a/sdk/python/cocoonsandbox/volumes.py +++ b/sdk/python/cocoonsandbox/volumes.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from typing import Any, Final, Literal, TypedDict, cast VolumePhase = Literal["attached", "pending", "detaching"] @@ -16,10 +16,11 @@ class _VolumeName(TypedDict): class Volume(_VolumeName, total=False): - """A catalog volume by name, with its guest mount point and an `rw` mode for writable claims.""" + """A catalog volume by name, with an optional mount, write mode, or caller-managed mounting.""" mount: str mode: str + attach_only: bool class _VolumeState(TypedDict): @@ -48,27 +49,65 @@ class SandboxVolumesResponse(_SandboxVolumesResponse, total=False): def _volume_response(reply: dict[str, Any] | None) -> SandboxVolumesResponse | None: if ( - reply is None - or not all(isinstance(reply.get(key), list) for key in ("volumes_want", "volumes", "states")) + not isinstance(reply, dict) + or not all( + isinstance(reply.get(key), list) and all(isinstance(entry, dict) for entry in reply[key]) + for key in ("volumes_want", "volumes", "states") + ) or not isinstance(reply.get("converging"), bool) ): return None - return cast(SandboxVolumesResponse, reply) + view = _sandbox_response(reply) + view["states"] = [dict(state) for state in reply["states"]] + return cast(SandboxVolumesResponse, view) + + +def _sandbox_response(reply: dict[str, Any]) -> dict[str, Any]: + view = dict(reply) + for key in ("volumes", "volumes_want"): + if isinstance(reply.get(key), list): + view[key] = _response_volumes(reply[key]) + return view + + +def _response_volumes(volumes: Iterable[Mapping[str, str | bool] | Volume]) -> list[Volume]: + # Response mounts identify raw devices; absent request mounts instead select a default path. + entries = [cast("Volume", dict(volume)) for volume in volumes] + for entry in entries: + if not entry.get("mount"): + entry["attach_only"] = True + return entries + + +def _volume_bodies( + volumes: Iterable[str | Mapping[str, str | bool] | Volume], mount: bool +) -> list[dict[str, str | bool]]: + if isinstance(volumes, (str, bytes, bytearray, Mapping)) or not isinstance(volumes, Iterable): + raise TypeError("volumes must be an iterable of volume names or mappings") + return [_volume_body(volume, mount) for volume in volumes] -def _volume_body(volume: str | Mapping[str, str] | Volume, mount: bool) -> dict[str, str]: +def _volume_body(volume: str | Mapping[str, str | bool] | Volume, mount: bool) -> dict[str, str | bool]: if isinstance(volume, str): return {"name": volume} if not isinstance(volume, Mapping): raise TypeError("volume must be a name string or mapping") - unknown = sorted(set(volume) - {"name", "mount", "mode"}) + unknown = sorted(set(volume) - {"name", "mount", "mode", "attach_only"}) if unknown: raise TypeError( - f"volume mapping accepts only name, mount, and mode, got unexpected key(s): {', '.join(unknown)}" + "volume mapping accepts only name, mount, mode, and attach_only, " + f"got unexpected key(s): {', '.join(unknown)}" ) + for field in ("name", "mount", "mode"): + if field in volume and not isinstance(volume[field], str): + raise TypeError(f"volume {field} must be a string") + if "attach_only" in volume and not isinstance(volume["attach_only"], bool): + raise TypeError("volume attach_only must be a boolean") + if volume.get("attach_only") and volume.get("mount"): + raise TypeError("volume mount is meaningless with attach_only=True, which leaves mounting to the caller") if not mount and "mount" in volume: raise TypeError("volume mount is meaningless with mount=False, which leaves mounting to the caller") - body = dict(cast(Mapping[str, str], volume)) + body = dict(cast("Mapping[str, str | bool]", volume)) mode = body.get("mode") if mode in (None, "", "ro"): body.pop("mode", None) diff --git a/sdk/python/examples/volume_controller.py b/sdk/python/examples/volume_controller.py new file mode 100644 index 00000000..4ffbcef7 --- /dev/null +++ b/sdk/python/examples/volume_controller.py @@ -0,0 +1,37 @@ +"""Compose volume detach and restore with controller-owned reservation and access hooks.""" + +from collections.abc import Callable +from dataclasses import dataclass + +from cocoonsandbox import Sandbox, Volume + + +@dataclass(frozen=True) +class VolumeHooks: + assert_reserved: Callable[[], None] + close_access: Callable[[], None] + quiesce_and_unmount: Callable[[], None] + mount_and_verify: Callable[[], None] + open_access: Callable[[], None] + + +def hibernate_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + hooks.quiesce_and_unmount() + view = sb.patch_volumes(remove=[name]) + if view["volumes_want"] or view["volumes"] or view["states"] or view["converging"]: + raise RuntimeError("volume references remain; keep access closed") + sb.hibernate() + + +def resume_volume(sb: Sandbox, name: str, hooks: VolumeHooks) -> None: + hooks.close_access() + hooks.assert_reserved() + sb.exec("true") + volume: Volume = {"name": name, "mode": "rw", "attach_only": True} + view = sb.patch_volumes(add=[volume]) + if view["volumes_want"] != [volume] or view["volumes"] != [volume] or view["converging"]: + raise RuntimeError("volume attachment is not settled; keep access closed") + hooks.mount_and_verify() + hooks.open_access() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 5a898e7d..0b87f3cd 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -13,9 +13,12 @@ readme = "README.md" [tool.setuptools.packages.find] include = ["cocoonsandbox*"] +[tool.pytest.ini_options] +pythonpath = ["."] + [tool.mypy] strict = true -files = ["cocoonsandbox"] +files = ["cocoonsandbox", "examples"] [tool.ruff] line-length = 120 diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 90b799dc..b40160c1 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -7,7 +7,7 @@ import pytest from conftest import FakeNode, sandbox_at -from cocoonsandbox import APIError, Client, Template +from cocoonsandbox import APIError, Client, Sandbox, Template class KeptNode(FakeNode): @@ -119,7 +119,10 @@ def test_claim_attaches_volumes_without_mounting(node): "volumes_attach_only": True, } ] - assert sb.volumes == [{"name": "imagenet"}, {"name": "scratch", "mode": "rw"}] + assert sb.volumes == [ + {"name": "imagenet", "attach_only": True}, + {"name": "scratch", "mode": "rw", "attach_only": True}, + ] def test_template_claim_attaches_volumes_without_mounting(node): @@ -127,7 +130,7 @@ def test_template_claim_attaches_volumes_without_mounting(node): sb = Template(Client(node), node, "task:v1", "none", "small").new(volumes=["imagenet"], mount=False) assert seen[0]["volumes_attach_only"] is True assert seen[0]["volumes"] == [{"name": "imagenet"}] - assert sb.volumes == [{"name": "imagenet"}] + assert sb.volumes == [{"name": "imagenet", "attach_only": True}] def test_claim_rejects_mount_without_mounting(node): @@ -382,3 +385,222 @@ def kept_node(): threading.Thread(target=server.serve_forever, daemon=True).start() yield f"127.0.0.1:{server.server_port}" server.shutdown() + + +@pytest.mark.parametrize("promoted", [False, True]) +def test_claim_rejects_mixed_mounting_before_request(node, promoted): + entries = [{"name": "mounted-ro"}, {"name": "device-rw", "mode": "rw", "attach_only": True}] + seen = recording_claim({"id": "sb_1", "token": "tok"}) + client = Client(node) + with pytest.raises(TypeError, match="claim without volumes"): + if promoted: + Template(client, node, "task:v1", "none", "small").new(volumes=entries) + else: + client.new("rt:24.04", volumes=entries) + assert seen == [] + + +@pytest.mark.parametrize("promoted", [False, True]) +@pytest.mark.parametrize("global_flag", [False, True]) +def test_claim_attach_only_uses_legacy_wire_encoding(node, promoted, global_flag): + entries = [{"name": "data", "attach_only": not global_flag}, {"name": "work", "mode": "rw", "attach_only": True}] + wire_entries = [{"name": "data"}, {"name": "work", "mode": "rw"}] + seen = recording_claim({"id": "sb_1", "token": "tok", "volumes": wire_entries}) + client = Client(node) + if promoted: + sb = Template(client, node, "task:v1", "none", "small").new(volumes=entries, mount=not global_flag) + else: + sb = client.new("rt:24.04", volumes=entries, mount=not global_flag) + assert seen[0]["volumes_attach_only"] is True + assert seen[0]["volumes"] == wire_entries + assert sb.volumes == [{**entry, "attach_only": True} for entry in wire_entries] + assert entries[0]["attach_only"] is not global_flag + + +@pytest.mark.parametrize("promoted", [False, True]) +def test_claim_rejects_per_entry_attach_only_mount(node, promoted): + entries = [{"name": "data", "mount": "/data", "attach_only": True}] + with pytest.raises(TypeError, match="meaningless with attach_only=True"): + if promoted: + Template(Client(node), node, "task:v1", "none", "small").new(volumes=entries) + else: + Client(node).new("rt:24.04", volumes=entries) + + +@pytest.mark.parametrize("promoted", [False, True]) +def test_claim_methods_reject_non_volume_sequences_before_request(node, promoted): + seen = recording_claim({"id": "sb_1", "token": "tok"}) + client = Client(node) + claim = ( + Template(client, node, "task:v1", "none", "small").new + if promoted + else lambda **kw: client.new("rt:24.04", **kw) + ) + for volumes in ("data", "", b"data", b"", bytearray(), {"name": "data"}): + with pytest.raises(TypeError): + claim(volumes=volumes) + assert seen == [] + + +@pytest.mark.parametrize("promoted", [False, True]) +@pytest.mark.parametrize("field", ["name", "mount", "mode"]) +def test_claim_methods_reject_non_string_volume_fields_before_request(node, promoted, field): + seen = recording_claim({"id": "sb_1", "token": "tok"}) + client = Client(node) + claim = ( + Template(client, node, "task:v1", "none", "small").new + if promoted + else lambda **kw: client.new("rt:24.04", **kw) + ) + volume = {"name": "data", field: True} + with pytest.raises(TypeError): + claim(volumes=[volume]) + assert seen == [] + + +@pytest.mark.parametrize("promoted", [False, True]) +def test_claim_methods_preserve_empty_volume_sequence_semantics(node, promoted): + seen = recording_claim({"id": "sb_1", "token": "tok"}) + client = Client(node) + claim = ( + Template(client, node, "task:v1", "none", "small").new + if promoted + else lambda **kw: client.new("rt:24.04", **kw) + ) + for volumes in (None, [], ()): + claim(volumes=volumes) + assert len(seen) == 3 + assert all("volumes" not in body for body in seen) + + +@pytest.mark.parametrize("promoted", [False, True]) +@pytest.mark.parametrize("kind", ["list", "tuple", "generator", "set", "empty-generator"]) +def test_claim_volume_iterables_are_consumed_once(node, promoted, kind): + seen = recording_claim({"id": "sb_1", "token": "tok"}) + consumed = [] + + def generate(): + if kind != "empty-generator": + consumed.append("data") + yield "data" + + volumes = {"list": ["data"], "tuple": ("data",), "set": {"data"}}.get(kind) + if volumes is None: + volumes = generate() + client = Client(node) + if promoted: + Template(client, node, "task:v1", "none", "small").new(volumes=volumes) + assert seen[0].get("no_redirect", False) is (kind == "empty-generator") + else: + client.new("rt:24.04", volumes=volumes) + if kind == "empty-generator": + assert "volumes" not in seen[0] + else: + assert seen[0]["volumes"] == [{"name": "data"}] + assert consumed == (["data"] if kind == "generator" else []) + + +def test_claim_and_summary_response_inference_copies_wire_data(node): + client = Client(node) + reply = {"id": "sb_1", "token": "tok", "volumes": [{"name": "raw"}, {"name": "mounted", "mount": "/data"}]} + FakeNode.routes[("POST", "/v1/claim")] = lambda body, path: (200, reply) + sb = client.new("rt:24.04") + assert sb.volumes[0]["attach_only"] is True + assert "attach_only" not in sb.volumes[1] + assert "attach_only" not in reply["volumes"][0] + sb.volumes[0]["name"] = "caller" + assert sb.volumes_want[0]["name"] == reply["volumes"][0]["name"] == "raw" + rows = [ + {"id": "sb_1", "volumes": [{"name": "raw"}], "volumes_want": [{"name": "raw", "mount": ""}]}, + {"id": "sb_2", "volumes": [], "volumes_want": []}, + {"id": "sb_3"}, + ] + FakeNode.routes[("GET", "/v1/sandboxes")] = lambda body, path: (200, {"sandboxes": rows}) + summaries = client.sandboxes() + assert summaries[0]["volumes"][0]["attach_only"] is True + assert summaries[0]["volumes_want"][0]["attach_only"] is True + assert summaries[1]["volumes"] == summaries[1]["volumes_want"] == [] + assert "volumes" not in summaries[2] + assert "attach_only" not in rows[0]["volumes"][0] + + +@pytest.mark.parametrize( + ("path", "payload"), + [ + ("/v1/checkpoints/ck_1/claim", {"volumes": [{"name": "raw"}]}), + ("/v1/sandboxes/sb_1/fork", {"children": [{"volumes": [{"name": "raw"}]}]}), + ("/v1/volumes", {"volumes": [{"name": "catalog"}]}), + ("/v1/info", {"metadata": {"volumes": [{"name": "opaque"}]}}), + ], +) +def test_transport_keeps_json_uninterpreted(spawn_node, path, payload): + addr = spawn_node({("GET", path): lambda body, request_path: (200, payload)}) + reply = Client(addr)._request(addr, "GET", path, None, "inspect") + assert reply == payload + assert "attach_only" not in repr(reply) + + +def test_claim_response_decoding_does_not_depend_on_route(spawn_node): + path = "/v1/custom-claims" + payload = {"id": "sb_1", "token": "tok", "volumes": [{"name": "raw"}]} + addr = spawn_node({("POST", path): lambda body, request_path: (200, payload)}) + sb = Client(addr)._claim_from(addr, {}, path=path) + assert sb.volumes == [{"name": "raw", "attach_only": True}] + assert "attach_only" not in payload["volumes"][0] + + +@pytest.mark.parametrize( + ("path", "fields"), + [ + ("/v1/claim", {"volumes": [None]}), + ("/v1/sandboxes", {"sandboxes": [None]}), + ("/v1/sandboxes/sb_1/fork", {"children": [None]}), + ("/v1/sandboxes/sb_1/volumes", {"volumes": ["bad"]}), + ], +) +def test_malformed_error_details_do_not_hide_http_error(spawn_node, path, fields): + payload = {"error": "storage unavailable", **fields} + addr = spawn_node({("GET", path): lambda body, request_path: (503, payload)}) + with pytest.raises(APIError) as caught: + Client(addr)._request(addr, "GET", path, None, "inspect") + assert caught.value.status == 503 + assert caught.value.message == "storage unavailable" + assert caught.value.reply == payload + + +@pytest.mark.parametrize("operation", ["checkpoint", "fork"]) +def test_sandbox_handle_decodes_volume_responses(spawn_node, operation): + payload = {"id": "sb_2", "token": "tok", "volumes": [{"name": "raw"}, {"name": "mounted", "mount": "/data"}]} + if operation == "checkpoint": + path = "/v1/checkpoints/ck_1/claim" + response = payload + else: + path = "/v1/sandboxes/sb_1/fork" + response = {"children": [payload]} + addr = spawn_node({("POST", path): lambda body, request_path: (200, response)}) + client = Client(addr) + sb = client.checkpoint("ck_1").new() if operation == "checkpoint" else sandbox_at(addr).fork(1)[0] + assert sb.volumes == [{"name": "raw", "attach_only": True}, {"name": "mounted", "mount": "/data"}] + assert sb.volumes_want == sb.volumes + assert "attach_only" not in payload["volumes"][0] + + +@pytest.mark.parametrize( + "volume", + [ + {"name": "data"}, + {"name": "data", "mount": ""}, + {"name": "data", "attach_only": False}, + {"name": "data", "attach_only": True}, + {"name": "data", "mount": "/data"}, + ], +) +def test_sandbox_constructor_preserves_volume_input(volume): + volumes = [volume] + sb = Sandbox(Client("unused.invalid:7777"), "sb_1", "tok", "unused.invalid:7777", volumes=volumes) + assert sb.volumes == sb.volumes_want == volumes + sb.volumes[0]["name"] = "caller" + assert sb.volumes_want == volumes + sb.volumes_want[0]["name"] = "desired" + assert volumes == [volume] + assert volume["name"] == "data" diff --git a/sdk/python/tests/test_volume_controller.py b/sdk/python/tests/test_volume_controller.py new file mode 100644 index 00000000..c100f3e9 --- /dev/null +++ b/sdk/python/tests/test_volume_controller.py @@ -0,0 +1,138 @@ +"""Run the documented controller functions through the SDK's HTTP boundary.""" + +import pytest +from conftest import FakeNode + +from cocoonsandbox import Client, VolumeError +from examples.volume_controller import VolumeHooks, hibernate_volume, resume_volume + +PATH = "/v1/sandboxes/sb_1" +EMPTY = {"volumes_want": [], "volumes": [], "states": [], "converging": False} +ATTACHED = { + "volumes_want": [{"name": "home", "mode": "rw"}], + "volumes": [{"name": "home", "mode": "rw"}], + "states": [{"name": "home", "state": "attached"}], + "converging": False, +} + + +@pytest.fixture +def controller(spawn_node, monkeypatch): + events = [] + replies = [] + fail = set() + gate = {"open": True} + + def hook(name): + def run(): + events.append(name) + if name == "close": + gate["open"] = False + if name in fail: + raise RuntimeError(name) + if name == "open": + gate["open"] = True + + return run + + def patch(body, path): + assert FakeNode.last_headers["Authorization"] == "Bearer root-token" + assert not gate["open"] + events.append(("patch", body)) + if replies: + return replies.pop(0) + return 200, EMPTY if "remove" in body else ATTACHED + + def hibernate(body, path): + assert FakeNode.last_headers["Authorization"] == "Bearer sandbox-token" + assert not gate["open"] + events.append("hibernate") + return 204, {} + + addr = spawn_node({("PATCH", PATH + "/volumes"): patch, ("POST", PATH + "/hibernate"): hibernate}) + client = Client(addr, api_token="root-token") + sb = client.attach(addr, "sb_1", "sandbox-token") + + def exec_true(*argv): + assert argv == ("true",) + assert not gate["open"] + events.append("wake") + return "" + + monkeypatch.setattr(sb, "exec", exec_true) + hooks = VolumeHooks(hook("reserved"), hook("close"), hook("unmount"), hook("verify"), hook("open")) + return sb, hooks, events, replies, fail, gate + + +def test_controller_cycle_uses_legacy_replies_and_opens_after_verification(controller): + sb, hooks, events, _, _, gate = controller + hibernate_volume(sb, "home", hooks) + assert not gate["open"] + resume_volume(sb, "home", hooks) + assert events == [ + "close", + "reserved", + "unmount", + ("patch", {"remove": ["home"]}), + "hibernate", + "close", + "reserved", + "wake", + ("patch", {"add": [{"name": "home", "mode": "rw", "attach_only": True}]}), + "verify", + "open", + ] + assert gate["open"] + assert sb.volumes == [{"name": "home", "mode": "rw", "attach_only": True}] + + +@pytest.mark.parametrize("failure", ["reserved", "unmount"]) +def test_controller_never_detaches_when_preconditions_fail(controller, failure): + sb, hooks, events, _, fail, gate = controller + fail.add(failure) + with pytest.raises(RuntimeError, match=failure): + hibernate_volume(sb, "home", hooks) + assert not gate["open"] + assert events == ["close", "reserved"] + (["unmount"] if failure == "unmount" else []) + + +def test_controller_failed_remove_needs_successful_retry_even_with_empty_view(controller): + sb, hooks, events, replies, _, gate = controller + replies.append((503, {**EMPTY, "error": "claim write failed"})) + with pytest.raises(VolumeError) as caught: + hibernate_volume(sb, "home", hooks) + assert caught.value.status == 503 + assert sb.volumes == sb.volumes_want == [] + assert not gate["open"] + assert "hibernate" not in events + + hibernate_volume(sb, "home", hooks) + assert [event for event in events if isinstance(event, tuple)] == [("patch", {"remove": ["home"]})] * 2 + assert events[-1] == "hibernate" + assert not gate["open"] + + +@pytest.mark.parametrize("remaining", ["volumes_want", "volumes", "states", "converging"]) +def test_controller_does_not_hibernate_an_unsettled_remove(controller, remaining): + sb, hooks, events, replies, _, gate = controller + replies.append((200, {**EMPTY, remaining: True if remaining == "converging" else [{"name": "other"}]})) + with pytest.raises(RuntimeError, match="references remain"): + hibernate_volume(sb, "home", hooks) + assert "hibernate" not in events + assert not gate["open"] + + +@pytest.mark.parametrize("failure", ["attach", "unsettled", "verify"]) +def test_controller_restore_failure_keeps_access_closed(controller, failure): + sb, hooks, events, replies, fail, gate = controller + if failure == "attach": + replies.append((400, {"error": "backing missing"})) + elif failure == "unsettled": + replies.append((200, {**ATTACHED, "converging": True})) + else: + fail.add("verify") + with pytest.raises(VolumeError if failure == "attach" else RuntimeError): + resume_volume(sb, "home", hooks) + assert not gate["open"] + assert "open" not in events + assert ("verify" in events) == (failure == "verify") diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 3226b82d..38bf4a43 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -46,8 +46,8 @@ def reply(body, path): @pytest.mark.parametrize("method", ["GET", "PUT", "PATCH"]) def test_volume_error_preserves_typed_partial_view(spawn_node, method): partial = { - "volumes_want": [{"name": "new", "mount": "/new"}], - "volumes": [{"name": "old", "mount": "/old"}], + "volumes_want": [{"name": "new", "mode": "rw"}], + "volumes": [{"name": "old"}], "converging": False, "states": [ {"name": "new", "state": "pending", "error": "retry"}, @@ -65,9 +65,14 @@ def test_volume_error_preserves_typed_partial_view(spawn_node, method): sb.patch_volumes(remove=["old"]) assert isinstance(caught.value, APIError) assert caught.value.status == 500 - assert caught.value.response == partial - assert sb.volumes_want == partial["volumes_want"] - assert sb.volumes == partial["volumes"] + expected = { + **partial, + "volumes_want": [{**partial["volumes_want"][0], "attach_only": True}], + "volumes": [{**partial["volumes"][0], "attach_only": True}], + } + assert caught.value.response == expected + assert sb.volumes_want == expected["volumes_want"] + assert sb.volumes == expected["volumes"] caught.value.response["volumes"][0]["name"] = "caller copy" assert sb.volumes[0]["name"] == "old" @@ -159,8 +164,154 @@ def test_incomplete_volume_view_preserves_handle(spawn_node, status): lambda sb: sb.set_volumes("data"), lambda sb: sb.patch_volumes(add="data"), lambda sb: sb.patch_volumes(remove="old"), + lambda sb: sb.patch_volumes(remove=b""), ], ) def test_volume_methods_reject_a_bare_string(black_hole, call): with pytest.raises(TypeError): call(sandbox_at(black_hole)) + + +def test_mixed_volume_modes_roundtrip(spawn_node): + entries = [ + {"name": "mounted-ro", "mount": "/volumes/mounted-ro"}, + {"name": "mounted-rw", "mount": "/work", "mode": "rw"}, + {"name": "device-ro", "attach_only": True}, + {"name": "device-rw", "mode": "rw", "attach_only": True}, + ] + view = {"volumes_want": entries, "volumes": entries, "states": [], "converging": False} + seen = [] + + def reply(body, path): + seen.append(body) + return 200, view + + sb = sandbox_at(spawn_node({("GET", PATH): reply, ("PUT", PATH): reply, ("PATCH", PATH): reply})) + got = sb.get_volumes() + sb.set_volumes(got["volumes_want"]) + sb.patch_volumes(add=got["volumes"]) + assert seen == [{}, {"volumes": entries}, {"add": entries}] + assert sb.volumes == sb.volumes_want == entries + + +@pytest.mark.parametrize("value", [None, 0, 1, "true", [], {}]) +@pytest.mark.parametrize("method", ["set", "patch"]) +def test_attach_only_requires_a_boolean(black_hole, value, method): + sb = sandbox_at(black_hole) + with pytest.raises(TypeError, match="attach_only must be a boolean"): + if method == "set": + sb.set_volumes([{"name": "data", "attach_only": value}]) + else: + sb.patch_volumes(add=[{"name": "data", "attach_only": value}]) + + +@pytest.mark.parametrize("method", ["set", "patch"]) +def test_attach_only_rejects_a_nonempty_mount(black_hole, method): + sb = sandbox_at(black_hole) + with pytest.raises(TypeError, match="meaningless with attach_only=True"): + if method == "set": + sb.set_volumes([{"name": "data", "mount": "/data", "attach_only": True}]) + else: + sb.patch_volumes(add=[{"name": "data", "mount": "/data", "attach_only": True}]) + + +def test_attach_only_empty_mount_and_false_flag_remain_explicit(spawn_node): + seen = [] + + def reply(body, path): + seen.append(body) + return 200, {"volumes_want": [], "volumes": [], "states": [], "converging": False} + + sb = sandbox_at(spawn_node({("PUT", PATH): reply})) + entries = [{"name": "device", "mount": "", "attach_only": True}, {"name": "mounted", "attach_only": False}] + sb.set_volumes(entries) + assert seen == [{"volumes": entries}] + + +@pytest.mark.parametrize("method", ["set", "patch"]) +@pytest.mark.parametrize("field", ["name", "mount", "mode"]) +def test_live_volume_methods_reject_non_string_fields_before_request(spawn_node, method, field): + seen = [] + + def reply(body, path): + seen.append(body) + return 200, {"volumes_want": [], "volumes": [], "states": [], "converging": False} + + sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) + volume = {"name": "data", field: True} + with pytest.raises(TypeError): + if method == "set": + sb.set_volumes([volume]) + else: + sb.patch_volumes(add=[volume]) + assert seen == [] + + +@pytest.mark.parametrize("method", ["set", "patch"]) +def test_live_volume_methods_reject_binary_collections_before_request(spawn_node, method): + seen = [] + + def reply(body, path): + seen.append(body) + return 200, {"volumes_want": [], "volumes": [], "states": [], "converging": False} + + sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) + for volumes in (b"data", b"", bytearray(), {"name": "data"}): + with pytest.raises(TypeError): + if method == "set": + sb.set_volumes(volumes) + else: + sb.patch_volumes(add=volumes) + assert seen == [] + + +@pytest.mark.parametrize("method", ["set", "patch"]) +@pytest.mark.parametrize("kind", ["list", "tuple", "generator", "set", "empty-generator"]) +def test_live_volume_iterables_are_consumed_once(spawn_node, method, kind): + seen = [] + + def reply(body, path): + seen.append(body) + return 200, {"volumes_want": [], "volumes": [], "states": [], "converging": False} + + consumed = [] + + def generate(): + if kind != "empty-generator": + consumed.append("data") + yield "data" + + volumes = {"list": ["data"], "tuple": ("data",), "set": {"data"}}.get(kind) + if volumes is None: + volumes = generate() + sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) + if method == "set": + sb.set_volumes(volumes) + else: + sb.patch_volumes(add=volumes) + entries = [] if kind == "empty-generator" else [{"name": "data"}] + assert seen == [{"volumes" if method == "set" else "add": entries}] + assert consumed == (["data"] if kind == "generator" else []) + + +@pytest.mark.parametrize("status", [200, 409]) +@pytest.mark.parametrize("field", ["volumes", "volumes_want", "states"]) +@pytest.mark.parametrize("entry", [None, "bad", []]) +def test_malformed_volume_view_preserves_error_and_handle(spawn_node, status, field, entry): + payload = {**VIEW, "error": "storage unavailable", field: [*VIEW[field], entry]} + sb = sandbox_at(spawn_node({("GET", PATH): lambda body, path: (status, payload)})) + sb.volumes = [{"name": "old"}] + sb.volumes_want = [{"name": "wanted"}] + with pytest.raises(APIError) as caught: + sb.get_volumes() + assert sb.volumes == [{"name": "old"}] + assert sb.volumes_want == [{"name": "wanted"}] + if status == 409: + assert isinstance(caught.value, VolumeError) + assert caught.value.status == 409 + assert caught.value.message == "storage unavailable" + assert caught.value.reply == payload + assert caught.value.response is None + else: + assert caught.value.status == 0 + assert caught.value.message == "reply without complete volume view"