Review all code using a newer model

.github/workflows/e2e.yaml

@@ -101,26 +101,13 @@ jobs:
     - name: Set up contain
       uses: solsson/setup-contain@49cc4cce1498df8a59e88fff52c1a9b747f11f08 # v1
 
-    - name: Build controller image (contain) with pinned tag
+    - name: Build controller image with pinned tag, assert the pinned digest
       env:
         T: ${{ steps.release.outputs.tag }}
-      run: |
-        set -euxo pipefail
-        CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-          -trimpath \
-          -buildvcs=false \
-          -ldflags "-s -w -X main.version=${T}" \
-          -o target/linux/amd64/buckety \
-          ./cmd/buckety
-        IMAGE="ghcr.io/yolean/buckety-controller:${T}" \
-          contain build --output ./oci --push=false
-
-    - name: Assert produced digest matches pinned
-      env:
         PINNED: ${{ steps.release.outputs.digest }}
       run: |
         set -euxo pipefail
-        BUILT=$(jq -r '.manifests[0].digest' oci/index.json)
+        BUILT=$(scripts/build-image.sh "$T" ./oci)
         if [[ "$BUILT" != "$PINNED" ]]; then
           echo "Pinned digest in deploy/kustomize/release/kustomization.yaml" >&2
           echo "  ${PINNED}" >&2
@@ -130,30 +117,13 @@ jobs:
           exit 1
         fi
 
-    # The driver-version scenario rotates the controller between a
-    # base, a patch-bumped and a major-bumped driver build (SPEC.md
-    # scenario #9, docs/SCAFFOLDING.md "Driver version stamping").
-    # These images exist only for that rotation: built here with
-    # per-driver -X flags and pushed straight to the run's local
-    # registry. They are not digest-pinned or published.
+    # The driver-version scenario rotates the controller through a
+    # patch-bumped and a major-bumped build of every driver.
+    # build-rotation-images.sh derives the versions from the binary,
+    # so this workflow knows nothing about drivers (SPEC §E2E
+    # harness #5). The images are not digest-pinned or published.
     - name: Build and push driver-version rotation images
-      run: |
-        set -euxo pipefail
-        for v in 0.1.1 1.0.0; do
-          CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-            -trimpath \
-            -buildvcs=false \
-            -ldflags "-s -w -X main.version=dv-${v} \
-              -X github.com/Yolean/buckety-controller/pkg/drivers/kadm.version=${v} \
-              -X github.com/Yolean/buckety-controller/pkg/drivers/s3.version=${v} \
-              -X github.com/Yolean/buckety-controller/pkg/drivers/gcs.version=${v}" \
-            -o target/linux/amd64/buckety \
-            ./cmd/buckety
-          IMAGE="ghcr.io/yolean/buckety-controller:dv-${v}" \
-            contain build --output "./oci-dv-${v}" --push=false
-          crane push --insecure "./oci-dv-${v}" \
-            "localhost:5000/yolean/buckety-controller:dv-${v}"
-        done
+      run: test/e2e/build-rotation-images.sh localhost:5000/yolean/buckety-controller
 
     - name: Run e2e harness
       env:
@@ -164,12 +134,9 @@ jobs:
         # hostname so node containerd can pull it.
         PUSH_AS: localhost:5000/yolean/buckety-controller:${{ steps.release.outputs.tag }}
         CONTROLLER_IMAGE: k3d-buckety-registry:5000/yolean/buckety-controller:${{ steps.release.outputs.tag }}
-        E2E_VERSION_BASE: 0.1.0
-        E2E_VERSION_PATCH: 0.1.1
-        E2E_VERSION_MAJOR: 1.0.0
         E2E_IMAGE_BASE: k3d-buckety-registry:5000/yolean/buckety-controller:${{ steps.release.outputs.tag }}
-        E2E_IMAGE_PATCH: k3d-buckety-registry:5000/yolean/buckety-controller:dv-0.1.1
-        E2E_IMAGE_MAJOR: k3d-buckety-registry:5000/yolean/buckety-controller:dv-1.0.0
+        E2E_IMAGE_PATCH: k3d-buckety-registry:5000/yolean/buckety-controller:dv-patch
+        E2E_IMAGE_MAJOR: k3d-buckety-registry:5000/yolean/buckety-controller:dv-major
       run: ./test/e2e/run.sh
 
     - name: Surface controller logs on failure
@@ -214,26 +181,13 @@ jobs:
     - name: Set up contain
       uses: solsson/setup-contain@49cc4cce1498df8a59e88fff52c1a9b747f11f08 # v1
 
-    - name: Rebuild controller image with pinned tag
+    - name: Rebuild controller image with pinned tag, assert the e2e-tested digest
       env:
         T: ${{ needs.e2e.outputs.tag }}
-      run: |
-        set -euxo pipefail
-        CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-          -trimpath \
-          -buildvcs=false \
-          -ldflags "-s -w -X main.version=${T}" \
-          -o target/linux/amd64/buckety \
-          ./cmd/buckety
-        IMAGE="ghcr.io/yolean/buckety-controller:${T}" \
-          contain build --output ./oci --push=false
-
-    - name: Assert publish-time digest matches the e2e-tested digest
-      env:
         PINNED: ${{ needs.e2e.outputs.digest }}
       run: |
         set -euxo pipefail
-        BUILT=$(jq -r '.manifests[0].digest' oci/index.json)
+        BUILT=$(scripts/build-image.sh "$T" ./oci)
         if [[ "$BUILT" != "$PINNED" ]]; then
           echo "publish rebuild produced ${BUILT}, e2e verified ${PINNED}" >&2
           exit 1

.gitignore

@@ -8,6 +8,7 @@
 # Test artefacts
 *.test
 *.out
+test/e2e/.work/
 
 # Go workspace
 go.work

README.md

@@ -4,8 +4,8 @@ A small in-cluster operator that provisions named resources on
 backing services (Kafka topics, S3 buckets) and mints
 `secretKeyRef`-friendly credentials Secrets for consumers.
 
-> **Status: v1alpha1 shipped.** Both drivers (`kadm`, `s3`) are
-> implemented and covered by the e2e suite in CI. Every commit on
+> **Status: v1alpha1 shipped.** Three drivers (`kadm`, `s3`,
+> `gcs`) are implemented and covered by the e2e suite in CI. Every commit on
 > `main` pins a reproducible image build in
 > [`deploy/kustomize/release/`](./deploy/kustomize/release), and
 > the same digest is pushed to `ghcr.io/yolean/buckety-controller`
@@ -45,8 +45,9 @@ here. The deviations are operational, not architectural:
 - **Recovery.** Standard Kubernetes status conditions
   (`Ready`, `Reconciling`, `BackendUnavailable`,
   `ParameterDrift`, `BlockedByAccesses`) cover the diagnostic
-  surface. Out-of-band drift on the backend is surfaced
-  explicitly rather than silently re-reconciled.
+  surface. Out-of-band drift on the backend is reapplied where
+  the driver can, and surfaced as `ParameterDrift` where it
+  cannot (a partition-count shrink, a bucket location).
 - **Portability.** Backend choice (VersityGW vs MinIO vs AWS S3)
   is a deploy-time cluster-maintainer decision, not part of the
   API. Consumer YAML moves between clusters with different
@@ -140,23 +141,24 @@ The base includes a `ValidatingWebhookConfiguration` whose CA is
 injected by cert-manager (`cert-manager.io/inject-ca-from`), and
 the webhook server needs a TLS Secret named
 `buckety-controller-webhook-tls`. On clusters without
-cert-manager the controller cannot start in its default shape.
-The cert-less recipe:
-
-1. Vendor `deploy/kustomize/crd/` and `deploy/kustomize/controller/`
-   separately (skip the composite base).
-2. Drop `webhook.yaml` from your controller overlay (or patch the
-   `ValidatingWebhookConfiguration` out).
-3. Pass `--enable-webhook=false` to the controller via a
-   Deployment args patch. No `Certificate`, no TLS Secret needed.
-
-In webhook-disabled mode, per-driver parameter and resolved-name
-validation runs in the reconcile loop and surfaces as
-`Ready=False` with reason `InvalidParameters` or `NameInvalid` on
-the resource's status (plus a Warning Event), instead of failing
-the apply. CRD-level CEL still enforces `spec.backend` /
-`spec.name` / `bucketyRef` / `credentialsSecretName` immutability
-and the `role` / `retentionPolicy` enums regardless.
+cert-manager, vendor `deploy/kustomize/release-tls-selfsigned/`
+instead of `release/`: it adds two `kube-webhook-certgen` Jobs
+that mint a self-signed serving certificate into that Secret and
+patch the webhook's `caBundle`, with no external infrastructure
+([`deploy/kustomize/webhook-certgen/README.md`](./deploy/kustomize/webhook-certgen/README.md)).
+The e2e suite deploys the controller this way.
+
+Running with `--enable-webhook=false` (and `webhook.yaml` dropped
+from the overlay) is a last resort, not a recommended shape:
+per-driver parameter and resolved-name validation then runs in
+the reconcile loop and surfaces as `Ready=False` with reason
+`InvalidParameters` or `NameInvalid` (plus a Warning Event)
+instead of failing the apply, and driver-level parameter
+immutability (a gcs bucket's `location`, its `serviceAccount`)
+is not enforced at all. CRD-level CEL still enforces
+`spec.backend` / `spec.name` / `bucketyRef` /
+`credentialsSecretName` immutability and the `role` /
+`retentionPolicy` enums regardless.
 
 ## Configure
 
@@ -494,7 +496,6 @@ The full list is in [`SPEC.md`](./SPEC.md). Highlights:
 - No MySQL driver.
 - No cross-namespace `bucketyRef`.
 - No hot-reload of `buckety-controller.yaml`.
-- No adoption of pre-existing backing resources.
 
 ## Filing issues
 

SPEC.md

@@ -261,7 +261,7 @@ spec:
   bucketyRef:
     name: orders
   credentialsSecretName: orders-reader
-  role: ReadOnly
+  role: Reader
   parameters:
     consumerGroupPrefix: "tenant1-orders-reporting-"
 
@@ -278,7 +278,7 @@ credentials drawn from the backend's root config. The CRD shape
 already supports per-access scoping for v1alpha2 — drivers are
 expected to ignore role/parameter values they don't yet
 implement and surface a `ScopingNotImplemented` condition rather
-than silently treating ReadOnly as ReadWrite.
+than silently treating Reader as ReadWrite.
 
 ## Implicit access (`defaultAccess`)
 
@@ -377,8 +377,8 @@ creation. Audit the fleet before adding one.
 
 Each driver carries a SemVer (`major.minor.patch`) advertised by
 the compiled binary (typically injected at build time via
-`-ldflags '-X main.driverVersion=...'`; see *Build and
-distribution*). At first reconcile the resource's
+`-ldflags '-X github.com/Yolean/buckety-controller/pkg/drivers/<driver>.version=...'`;
+see *Build and distribution*). At first reconcile the resource's
 `status.driverMajor` is stamped from this version and is
 **sticky** thereafter, exactly like `status.backend`. The full
 running version is mirrored to `status.driverBuildVersion` on
@@ -644,10 +644,12 @@ call, guided by the chosen operator SDK.
   Idempotent for the same `(Buckety, BucketyAccess)` pair.
 - `RevokeAccess` — removes the backend-side principal (no-op in
   v1alpha1), idempotent on NotFound.
-- `ValidateParameters` — used by the admission webhook; returns
-  a per-key error map.
-- `Version()` — returns the driver's SemVer for
-  `status.driverVersion` stamping.
+- `ValidateParameters` — used by the admission webhook and, with
+  the webhook disabled, by the reconciler; returns an error
+  naming the offending key.
+- `Version()` — returns the driver's SemVer, from which
+  `status.driverMajor` (sticky) and `status.driverBuildVersion`
+  are stamped.
 - Driver also exposes its OpenAPI schemas separately (see
   Schemas).
 
@@ -667,13 +669,21 @@ specified:
   (Kafka partition count can grow but not shrink), the driver
   surfaces `ParameterDrift` and waits for human resolution.
 - `status.observedGeneration` advances when `metadata.generation`
-  has been fully reconciled, including downstream Secret state
-  (not just the backend resource). Until then,
-  `status.observedGeneration` lags `metadata.generation` and
-  `Ready=False`.
-- The controller MUST be honest in `status.conditions`:
-  `Ready=False` until both the backend resource AND any
-  access-side Secrets are in sync.
+  has been fully reconciled. Until then it lags
+  `metadata.generation` and `Ready=False`.
+- The controller MUST be honest in `status.conditions`. A
+  `Buckety` is `Ready` when its backend resource matches the spec
+  and its implicit access (if any) has been materialised; the
+  access-side Secret has its own `Ready` on the `BucketyAccess`.
+  The two cannot depend on each other: an access waits for its
+  `Buckety` to be `Ready` before minting, so a `Buckety` that
+  waited for the Secret would never get there.
+- Condition messages may lag the latest attempt: a status patch
+  is written only when something other than a message changed,
+  because a message carrying a per-attempt token (a provider's
+  request or error id) would otherwise re-trigger the controller
+  through its own watch at the provider's answer rate. The
+  condition's status and reason are always current.
 
 ## Off the data path
 
@@ -816,12 +826,18 @@ one user-managed key for it — native GCS auth with bucket-scoped
 blast radius, alongside the (still backend-wide) HMAC pair. The
 create-only-retrievable-key problem that deferred per-access
 minting is solved by `GrantRequest.ExistingSecretData`: GrantAccess
-returns the Secret's current key unchanged while it still verifies
-against `keys.list`, and mints only when the key is absent, revoked
-out of band, or expired — which makes server-side key deletion the
-manual rotation runbook. `status.principal` is the key's full
-resource name and `RevokeAccess` deletes it, so BucketyAccess
-deletion performs real revocation for these Secrets.
+returns the Secret's current key unchanged, and mints only when
+the Secret has none or the key is positively unusable (expired
+per `keys.list`, or not this SA's). Absence from `keys.list` is
+deliberately NOT a mint trigger: the listing is eventually
+consistent and misses a key minted a second earlier, which is
+when the follow-up reconcile runs - and a mint revokes the
+replaced key, so it must rest on evidence rather than on a
+missing entry. The manual rotation runbook is therefore to delete
+the Secret: the next reconcile mints a fresh key and revokes the
+previous one. `status.principal` is the key's full resource name
+and `RevokeAccess` deletes it, so BucketyAccess deletion performs
+real revocation for these Secrets.
 
 The `serviceAccounts.project` SHOULD be a dedicated identity
 project, separate from the bucket project: key creation equals
@@ -1136,7 +1152,7 @@ Required CI matrix for v1alpha1:
 
 Adding an implementation later (e.g. AWS S3 once the project has
 credentials and a budget) requires no example or harness changes,
-only a new controller-config overlay under `test/e2e/overlays/`
+only a new controller-config overlay under `test/e2e/configs/`
 and the corresponding GHA secret.
 
 ## Build and distribution
@@ -1175,7 +1191,6 @@ and the corresponding GHA secret.
   Secret output > gcs driver), but role scoping remains
   unimplemented.
 - Cross-namespace `bucketyRef`.
-- Adopting backing resources that already exist outside Buckety.
 - Quota enforcement.
 - Hot-reload of `buckety-controller.yaml` (envsubst is
   startup-only; rotating credentials requires re-rolling the
@@ -1186,8 +1201,8 @@ and the corresponding GHA secret.
   machinery is already rotation-shaped, and scheduled key
   rotation (mint new, overlap one period, garbage-collect the
   previous key) is the planned follow-up; until then rotation is
-  operator-driven - delete the key server-side and the next
-  reconcile re-mints (`gcloud iam service-accounts keys delete`).
+  operator-driven - delete the access Secret and the next
+  reconcile mints a fresh key and revokes the previous one.
 - Multi-cluster federation.
 - Admission webhook for cross-resource invariants. Per-resource
   parameter validation (against per-driver schemas) and
@@ -1217,7 +1232,18 @@ contract above:
 5. **Status conditions catalog.** Minimum required: `Ready`,
    `Reconciling`, `BackendUnavailable`, `DriverVersionIncompatible`,
    `ParameterDrift`, `BlockedByAccesses`, `ScopingNotImplemented`.
-   Maintainer can add more.
+   Maintainer can add more. Implemented `Ready=False` reasons
+   beyond the failure ones: `Provisioning` (the backend needs a
+   moment, e.g. IAM propagation of a fresh service account;
+   prompt requeue, bounded), `DeletingContents` and
+   `RevokingAccesses` (teardown in progress), `SecretConflict`
+   (the credentials Secret exists and is not ours),
+   `BackendResourceExists` (adoption refused), `InspectFailed`.
+   Events beyond condition transitions: `Adopted`,
+   `RetainedOnDelete`, `DeletionBlocked`, and `PrincipalReplaced`
+   on a `BucketyAccess` whose credential was re-minted and the
+   previous one revoked - the signal to restart consumers that
+   loaded the Secret once.
 
 ## What to do before writing code