From 4dc57a100d8a2ae50c7f7c7516274b7b651722e1 Mon Sep 17 00:00:00 2001
From: Marcel <156897072+mblos@users.noreply.github.com>
Date: Fri, 9 Oct 2026 11:13:29 +0200
Subject: [PATCH 1/9] fix: self-heal workqueue dedup stale-read in Ready branch
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When Phase 6 writes a new parentGen concurrently with a Reservation
reconcile, the workqueue deduplicates the watch event if the key is
already pending. The reconcile then reads a stale version
(parentGen==obs, no echo fires) and returns nil — leaving the CR stuck
for 20s until the next natural requeue.

Fix: after the echo check, cross-check the parent CR's .Generation. If
parentGen < cr.Generation, return ctrl.Result{Requeue: true} so the
rate-limited queue retries until Get() returns the post-Phase-6 version.

Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com>
---
 .../commitments/reservation_controller.go     | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go
index 8c19e1d71..e260c72e5 100644
--- a/internal/scheduling/reservations/commitments/reservation_controller.go
+++ b/internal/scheduling/reservations/commitments/reservation_controller.go
@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"strings"
 	"time"
 
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -168,6 +169,26 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr
 			}
 		}
 
+		// Stale-read guard: workqueue dedup can cause this reconcile to read a
+		// pre-Phase-6 version of the Reservation (parentGen==obs, but Phase 6 already
+		// wrote a newer parentGen at a higher resourceVersion whose watch event was
+		// dropped). Cross-check the parent CR's generation to detect this and requeue
+		// with exponential backoff so the next Get() returns the fresh version.
+		if res.Spec.CommittedResourceReservation != nil {
+			if idx := strings.LastIndex(res.Name, "-"); idx > 0 {
+				var parentCR v1alpha1.CommittedResource
+				if err := r.Get(ctx, client.ObjectKey{Name: res.Name[:idx]}, &parentCR); err == nil {
+					if res.Spec.CommittedResourceReservation.ParentGeneration < parentCR.Generation {
+						logger.V(1).Info("stale reservation: parentGen behind parent CR generation, requeuing",
+							"parentGen", res.Spec.CommittedResourceReservation.ParentGeneration,
+							"crGen", parentCR.Generation,
+						)
+						return ctrl.Result{Requeue: true}, nil
+					}
+				}
+			}
+		}
+
 		// Verify all allocations in Spec against actual VM state
 		result, err := r.reconcileAllocations(ctx, &res)
 		if err != nil {

From 990019a6393253e285e3d8607f2e457a2ed7e28e Mon Sep 17 00:00:00 2001
From: Marcel <156897072+mblos@users.noreply.github.com>
Date: Mon, 12 Oct 2026 08:55:06 +0200
Subject: [PATCH 2/9] fix linter

Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com>
---
 .../reservations/commitments/reservation_controller.go          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go
index e260c72e5..18b3307b0 100644
--- a/internal/scheduling/reservations/commitments/reservation_controller.go
+++ b/internal/scheduling/reservations/commitments/reservation_controller.go
@@ -183,7 +183,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr
 							"parentGen", res.Spec.CommittedResourceReservation.ParentGeneration,
 							"crGen", parentCR.Generation,
 						)
-						return ctrl.Result{Requeue: true}, nil
+						return ctrl.Result{RequeueAfter: 100 * time.Millisecond}, nil
 					}
 				}
 			}

From adb4613e1376a41f7a76c089b667b279611d39d7 Mon Sep 17 00:00:00 2001
From: Marcel <156897072+mblos@users.noreply.github.com>
Date: Mon, 12 Oct 2026 09:04:55 +0200
Subject: [PATCH 3/9] let the cr controller notice when res controller needs
 another watch trigger to update reservations

Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com>
---
 .../committed_resource_controller.go          | 22 +++++++++++++++++++
 .../commitments/reservation_controller.go     | 21 ------------------
 2 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go
index 6d6d4b443..da09650c6 100644
--- a/internal/scheduling/reservations/commitments/committed_resource_controller.go
+++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go
@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strconv"
 	"time"
 
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -432,6 +433,27 @@ func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Co
 		)
 		if res.Status.CommittedResourceReservation == nil ||
 			res.Status.CommittedResourceReservation.ObservedParentGeneration != cr.Generation {
+			// parentGen has been written to the Reservation spec (syncReservationMetadata
+			// already ran for this generation) but the Reservation controller hasn't echoed
+			// it to status yet. This should be rare — it means the Reservation controller's
+			// watch event for that spec write was dropped by workqueue deduplication. Poke
+			// the Reservation to generate a fresh watch event so it wakes up and echoes.
+			if res.Spec.CommittedResourceReservation != nil &&
+				res.Spec.CommittedResourceReservation.ParentGeneration == cr.Generation {
+				ctrl.LoggerFrom(ctx).Info("nudging stale reservation: parentGen written to spec but echo pending",
+					"slot", res.Name,
+					"crGen", cr.Generation,
+					"obs", obs,
+				)
+				old := res.DeepCopy()
+				if res.Annotations == nil {
+					res.Annotations = map[string]string{}
+				}
+				res.Annotations["cortex.cloud/echo-nudge"] = strconv.FormatInt(time.Now().UnixMilli(), 10)
+				if pErr := r.Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(pErr) != nil {
+					ctrl.LoggerFrom(ctx).V(1).Info("failed to nudge stale reservation echo", "slot", res.Name, "err", pErr)
+				}
+			}
 			allReady = false
 			continue
 		}
diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go
index 18b3307b0..8c19e1d71 100644
--- a/internal/scheduling/reservations/commitments/reservation_controller.go
+++ b/internal/scheduling/reservations/commitments/reservation_controller.go
@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
-	"strings"
 	"time"
 
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -169,26 +168,6 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr
 			}
 		}
 
-		// Stale-read guard: workqueue dedup can cause this reconcile to read a
-		// pre-Phase-6 version of the Reservation (parentGen==obs, but Phase 6 already
-		// wrote a newer parentGen at a higher resourceVersion whose watch event was
-		// dropped). Cross-check the parent CR's generation to detect this and requeue
-		// with exponential backoff so the next Get() returns the fresh version.
-		if res.Spec.CommittedResourceReservation != nil {
-			if idx := strings.LastIndex(res.Name, "-"); idx > 0 {
-				var parentCR v1alpha1.CommittedResource
-				if err := r.Get(ctx, client.ObjectKey{Name: res.Name[:idx]}, &parentCR); err == nil {
-					if res.Spec.CommittedResourceReservation.ParentGeneration < parentCR.Generation {
-						logger.V(1).Info("stale reservation: parentGen behind parent CR generation, requeuing",
-							"parentGen", res.Spec.CommittedResourceReservation.ParentGeneration,
-							"crGen", parentCR.Generation,
-						)
-						return ctrl.Result{RequeueAfter: 100 * time.Millisecond}, nil
-					}
-				}
-			}
-		}
-
 		// Verify all allocations in Spec against actual VM state
 		result, err := r.reconcileAllocations(ctx, &res)
 		if err != nil {

From fec3c7150a010473b391caa4013225c2cf417bd4 Mon Sep 17 00:00:00 2001
From: Marcel <156897072+mblos@users.noreply.github.com>
Date: Mon, 12 Oct 2026 09:06:39 +0200
Subject: [PATCH 4/9] test

Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com>
---
 .../committed_resource_controller_test.go     | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go
index 19730f432..981d17f75 100644
--- a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go
+++ b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go
@@ -1170,6 +1170,7 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) {
 		wantReadySlots  int
 		wantReason      string
 		wantFailedSlots []string
+		wantNudge       bool
 	}{
 		{
 			name:           "Ready=False at stale generation: treated as pending",
@@ -1179,6 +1180,7 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) {
 			wantAllReady:   false,
 			wantAnyFailed:  false,
 			wantReadySlots: 0,
+			wantNudge:      true,
 		},
 		{
 			name:            "Ready=False at current generation: is a current failure",
@@ -1206,6 +1208,15 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) {
 			wantAnyFailed:  false,
 			wantReadySlots: 0,
 		},
+		{
+			name:           "parentGen written to spec but echo not yet in status: nudge annotation set",
+			obsGen:         1, // obs behind cr.Generation=2; spec.parentGeneration already set to cr.Generation
+			condStatus:     metav1.ConditionTrue,
+			wantAllReady:   false,
+			wantAnyFailed:  false,
+			wantReadySlots: 0,
+			wantNudge:      true,
+		},
 	}
 
 	for _, tt := range tests {
@@ -1267,6 +1278,14 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) {
 			if !slices.Equal(failedSlots, tt.wantFailedSlots) {
 				t.Errorf("failedSlots: want %v, got %v", tt.wantFailedSlots, failedSlots)
 			}
+			var updated v1alpha1.Reservation
+			if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: child.Name}, &updated); err != nil {
+				t.Fatalf("get updated reservation: %v", err)
+			}
+			_, hasNudge := updated.Annotations["cortex.cloud/echo-nudge"]
+			if hasNudge != tt.wantNudge {
+				t.Errorf("echo-nudge annotation: want %v, got %v", tt.wantNudge, hasNudge)
+			}
 		})
 	}
 }

From d679b1acb11077c8c5f6c9fb9e086c7fdb766a30 Mon Sep 17 00:00:00 2001
From: Marcel <156897072+mblos@users.noreply.github.com>
Date: Mon, 12 Oct 2026 09:28:41 +0200
Subject: [PATCH 5/9] fix: response with bad request when flavor group does not
 accept commitment

Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com>
---
 .../reservations/commitments/api/change_commitments.go         | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/internal/scheduling/reservations/commitments/api/change_commitments.go b/internal/scheduling/reservations/commitments/api/change_commitments.go
index f181a0fbe..0aec12167 100644
--- a/internal/scheduling/reservations/commitments/api/change_commitments.go
+++ b/internal/scheduling/reservations/commitments/api/change_commitments.go
@@ -274,7 +274,8 @@ ProcessLoop:
 			}
 			if !handlesCommitments {
 				failedReason = fmt.Sprintf("flavor group %q is not configured to handle %s commitments", flavorGroupName, resourceType)
-				sanitisedReason = "internal error processing request"
+				sanitisedReason = failedReason
+				isBadRequest = true
 				rollback = true
 				break ProcessLoop
 			}

From c016d6dbbbfc665a26ffb9fd8884bf06f3ef34ef Mon Sep 17 00:00:00 2001
From: Marcel <156897072+mblos@users.noreply.github.com>
Date: Mon, 12 Oct 2026 11:00:19 +0200
Subject: [PATCH 6/9] refactor

Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com>
---
 .../committed_resource_controller.go          | 82 ++++++++++++-----
 .../committed_resource_controller_test.go     | 87 ++++++++++++++++++-
 2 files changed, 146 insertions(+), 23 deletions(-)

diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go
index da09650c6..89b6f3ac1 100644
--- a/internal/scheduling/reservations/commitments/committed_resource_controller.go
+++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go
@@ -125,7 +125,12 @@ func (r *CommittedResourceController) reconcilePending(ctx context.Context, logg
 		// error would cause controller-runtime to ignore RequeueAfter and use exponential backoff.
 		return ctrl.Result{RequeueAfter: delay}, nil
 	}
-	allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots)
+	// alreadyReserving is true when the CR condition already shows Reserving for this generation,
+	// meaning at least one reconcile cycle (~1s) has elapsed since we first entered the waiting state.
+	// We only nudge stale Reservation slots after that delay to give the normal echo path a chance
+	// to complete first, keeping logs quiet in the happy path.
+	alreadyReserving := isAlreadyReserving(cr)
+	allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots, alreadyReserving)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
@@ -152,7 +157,13 @@ func (r *CommittedResourceController) reconcilePending(ctx context.Context, logg
 		// gate should not fire while we wait for slots to become ready.
 		// Fallback requeue: in case the watch event is missed,
 		// re-check after 1s rather than waiting for the full watch timeout.
-		if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, "waiting for reservation placement", true); err != nil {
+		// Use "still waiting" message after the first cycle so the condition is a visible hint
+		// that a nudge has been (or is about to be) sent to stale slots.
+		msg := "waiting for reservation placement"
+		if alreadyReserving {
+			msg = "still waiting for reservation placement"
+		}
+		if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, msg, true); err != nil {
 			return ctrl.Result{}, err
 		}
 		return ctrl.Result{RequeueAfter: time.Second}, nil
@@ -184,7 +195,7 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo
 			if applyErr != nil {
 				logger.V(1).Info("spurious-rejection recovery: apply failed, will retry", "err", applyErr)
 			} else {
-				allReady, anyFailed, _, _, _, _, checkErr := r.checkChildReservationStatus(ctx, cr, result.TotalSlots)
+				allReady, anyFailed, _, _, _, _, checkErr := r.checkChildReservationStatus(ctx, cr, result.TotalSlots, false)
 				if checkErr == nil && allReady && !anyFailed {
 					logger.Info("recovered from spurious rejection: all slots ready", "generation", cr.Generation)
 					return ctrl.Result{}, r.setAccepted(ctx, cr)
@@ -223,7 +234,8 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo
 		// error would cause controller-runtime to ignore RequeueAfter and use exponential backoff.
 		return ctrl.Result{RequeueAfter: delay}, nil
 	}
-	allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots)
+	alreadyReserving := isAlreadyReserving(cr)
+	allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots, alreadyReserving)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
@@ -248,7 +260,13 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo
 		// gate should not fire while we wait for slots to become ready.
 		// Fallback requeue: in case the watch event is missed,
 		// re-check after 1s rather than waiting for the full watch timeout.
-		if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, "waiting for reservation placement", true); err != nil {
+		// Use "still waiting" message after the first cycle so the condition is a visible hint
+		// that a nudge has been (or is about to be) sent to stale slots.
+		msg := "waiting for reservation placement"
+		if alreadyReserving {
+			msg = "still waiting for reservation placement"
+		}
+		if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, msg, true); err != nil {
 			return ctrl.Result{}, err
 		}
 		return ctrl.Result{RequeueAfter: time.Second}, nil
@@ -399,7 +417,7 @@ func (r *CommittedResourceController) applyReservationState(ctx context.Context,
 // Returns allReady=false, anyFailed=false when some children have no condition yet (placement pending).
 // failReason includes a slot count when more than one slot failed.
 // failedSlots lists the names of all failing reservation slots for log context.
-func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Context, cr *v1alpha1.CommittedResource, expectedSlots int) (allReady, anyFailed bool, readySlots int, failReason, firstFailMessage string, failedSlots []string, err error) {
+func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Context, cr *v1alpha1.CommittedResource, expectedSlots int, shouldNudge bool) (allReady, anyFailed bool, readySlots int, failReason, firstFailMessage string, failedSlots []string, err error) {
 	var list v1alpha1.ReservationList
 	if err := r.List(ctx, &list,
 		client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource},
@@ -435,23 +453,31 @@ func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Co
 			res.Status.CommittedResourceReservation.ObservedParentGeneration != cr.Generation {
 			// parentGen has been written to the Reservation spec (syncReservationMetadata
 			// already ran for this generation) but the Reservation controller hasn't echoed
-			// it to status yet. This should be rare — it means the Reservation controller's
-			// watch event for that spec write was dropped by workqueue deduplication. Poke
-			// the Reservation to generate a fresh watch event so it wakes up and echoes.
-			if res.Spec.CommittedResourceReservation != nil &&
+			// it to status yet. Poke the Reservation to generate a fresh watch event so the
+			// Reservation controller wakes up and fires the echo. Only nudge once per CR
+			// generation: if the annotation already carries the current generation we already
+			// sent the watch event — skip silently and let the 1s fallback requeue do the
+			// waiting. This bounds the Info log to at most one occurrence per slot per generation.
+			// shouldNudge is false on the first cycle (CR condition not yet set for this
+			// generation) to give the normal echo path ~1s to complete before intervening.
+			if shouldNudge &&
+				res.Spec.CommittedResourceReservation != nil &&
 				res.Spec.CommittedResourceReservation.ParentGeneration == cr.Generation {
-				ctrl.LoggerFrom(ctx).Info("nudging stale reservation: parentGen written to spec but echo pending",
-					"slot", res.Name,
-					"crGen", cr.Generation,
-					"obs", obs,
-				)
-				old := res.DeepCopy()
-				if res.Annotations == nil {
-					res.Annotations = map[string]string{}
-				}
-				res.Annotations["cortex.cloud/echo-nudge"] = strconv.FormatInt(time.Now().UnixMilli(), 10)
-				if pErr := r.Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(pErr) != nil {
-					ctrl.LoggerFrom(ctx).V(1).Info("failed to nudge stale reservation echo", "slot", res.Name, "err", pErr)
+				currentGen := strconv.FormatInt(cr.Generation, 10)
+				if res.Annotations["cortex.cloud/echo-nudge"] != currentGen {
+					ctrl.LoggerFrom(ctx).Info("nudging stale reservation: parentGen written to spec but echo pending",
+						"slot", res.Name,
+						"crGen", cr.Generation,
+						"obs", obs,
+					)
+					old := res.DeepCopy()
+					if res.Annotations == nil {
+						res.Annotations = map[string]string{}
+					}
+					res.Annotations["cortex.cloud/echo-nudge"] = currentGen
+					if pErr := r.Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(pErr) != nil {
+						ctrl.LoggerFrom(ctx).V(1).Info("failed to nudge stale reservation echo", "slot", res.Name, "err", pErr)
+					}
 				}
 			}
 			allReady = false
@@ -594,6 +620,18 @@ func isRejectedForGeneration(cr *v1alpha1.CommittedResource) bool {
 		cond.ObservedGeneration == cr.Generation
 }
 
+// isAlreadyReserving returns true when the CR condition already shows Reserving for this
+// generation, meaning at least one reconcile cycle (~1s) has elapsed since we first entered
+// the waiting-for-slots state. Used to gate the echo-nudge so the first reconcile cycle is
+// always silent, giving the normal Reservation echo path a chance to complete first.
+func isAlreadyReserving(cr *v1alpha1.CommittedResource) bool {
+	cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady)
+	return cond != nil &&
+		cond.Status == metav1.ConditionFalse &&
+		cond.Reason == v1alpha1.CommittedResourceReasonReserving &&
+		cond.ObservedGeneration == cr.Generation
+}
+
 // retryDelay computes an exponential backoff interval for the AllowRejection=false retry paths.
 // The exponent is derived from the time already spent in Reserving state: each doubling period
 // advances one step, giving the same base→2*base→4*base→… sequence as a counter would, but
diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go
index 981d17f75..d5732f0d0 100644
--- a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go
+++ b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go
@@ -1157,6 +1157,74 @@ func TestCommittedResourceController_BadSpec(t *testing.T) {
 // Tests: checkChildReservationStatus generation guard
 // ============================================================================
 
+// TestCommittedResourceController_NudgeDelay verifies that the echo-nudge is not sent on
+// the first reconcile cycle (when the condition is not yet Reserving for this generation),
+// but is sent on the second cycle — and that the condition message reflects the cycle.
+func TestCommittedResourceController_NudgeDelay(t *testing.T) {
+	scheme := newCRTestScheme(t)
+	cr := newTestCommittedResource("test-cr", v1alpha1.CommitmentStatusPending)
+	cr.Spec.AllowRejection = false
+	k8sClient := newCRTestClient(scheme, cr, newTestFlavorKnowledge())
+	controller := &CommittedResourceController{Client: k8sClient, Scheme: scheme}
+
+	getCR := func() v1alpha1.CommittedResource {
+		t.Helper()
+		var updated v1alpha1.CommittedResource
+		if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: cr.Name}, &updated); err != nil {
+			t.Fatalf("get CR: %v", err)
+		}
+		return updated
+	}
+	getChildAnnotation := func() string {
+		t.Helper()
+		var list v1alpha1.ReservationList
+		if err := k8sClient.List(context.Background(), &list, client.MatchingLabels{
+			v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource,
+		}); err != nil {
+			t.Fatalf("list reservations: %v", err)
+		}
+		for _, res := range list.Items {
+			if res.Spec.CommittedResourceReservation != nil &&
+				res.Spec.CommittedResourceReservation.CommitmentUUID == cr.Spec.CommitmentUUID {
+				return res.Annotations["cortex.cloud/echo-nudge"]
+			}
+		}
+		return ""
+	}
+
+	// Cycle 1: no Reserving condition exists yet → isAlreadyReserving=false → no nudge,
+	// condition message = "waiting for reservation placement".
+	if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil {
+		t.Fatalf("reconcile 1: %v", err)
+	}
+	cond1 := meta.FindStatusCondition(getCR().Status.Conditions, v1alpha1.CommittedResourceConditionReady)
+	if cond1 == nil || cond1.Reason != v1alpha1.CommittedResourceReasonReserving {
+		t.Fatalf("cycle 1: expected Reserving condition, got %+v", cond1)
+	}
+	if cond1.Message != "waiting for reservation placement" {
+		t.Errorf("cycle 1: message want %q, got %q", "waiting for reservation placement", cond1.Message)
+	}
+	if got := getChildAnnotation(); got != "" {
+		t.Errorf("cycle 1: echo-nudge annotation should not be set, got %q", got)
+	}
+
+	// Cycle 2: Reserving condition is now set for this generation → isAlreadyReserving=true →
+	// nudge fires, condition message = "still waiting for reservation placement".
+	if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil {
+		t.Fatalf("reconcile 2: %v", err)
+	}
+	cond2 := meta.FindStatusCondition(getCR().Status.Conditions, v1alpha1.CommittedResourceConditionReady)
+	if cond2 == nil || cond2.Reason != v1alpha1.CommittedResourceReasonReserving {
+		t.Fatalf("cycle 2: expected Reserving condition, got %+v", cond2)
+	}
+	if cond2.Message != "still waiting for reservation placement" {
+		t.Errorf("cycle 2: message want %q, got %q", "still waiting for reservation placement", cond2.Message)
+	}
+	if got := getChildAnnotation(); got == "" {
+		t.Errorf("cycle 2: echo-nudge annotation should be set")
+	}
+}
+
 // TestCheckChildReservationStatus_GenerationGuard verifies the single-pass logic that
 // distinguishes a stale Ready=False (previous generation) from a current failure.
 func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) {
@@ -1165,6 +1233,7 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) {
 		obsGen          int64
 		condStatus      metav1.ConditionStatus // "" = no condition set
 		condMessage     string
+		shouldNudge     bool
 		wantAllReady    bool
 		wantAnyFailed   bool
 		wantReadySlots  int
@@ -1177,16 +1246,29 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) {
 			obsGen:         1,
 			condStatus:     metav1.ConditionFalse,
 			condMessage:    "no hosts available",
+			shouldNudge:    true,
 			wantAllReady:   false,
 			wantAnyFailed:  false,
 			wantReadySlots: 0,
 			wantNudge:      true,
 		},
+		{
+			name:           "Ready=False at stale generation: no nudge on first cycle",
+			obsGen:         1,
+			condStatus:     metav1.ConditionFalse,
+			condMessage:    "no hosts available",
+			shouldNudge:    false,
+			wantAllReady:   false,
+			wantAnyFailed:  false,
+			wantReadySlots: 0,
+			wantNudge:      false,
+		},
 		{
 			name:            "Ready=False at current generation: is a current failure",
 			obsGen:          2,
 			condStatus:      metav1.ConditionFalse,
 			condMessage:     "no hosts available",
+			shouldNudge:     true,
 			wantAllReady:    false,
 			wantAnyFailed:   true,
 			wantReadySlots:  0,
@@ -1197,6 +1279,7 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) {
 			name:           "Ready=True at current generation: allReady",
 			obsGen:         2,
 			condStatus:     metav1.ConditionTrue,
+			shouldNudge:    true,
 			wantAllReady:   true,
 			wantReadySlots: 1,
 		},
@@ -1204,6 +1287,7 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) {
 			name:           "no condition yet at current generation: still pending",
 			obsGen:         2,
 			condStatus:     "", // no condition
+			shouldNudge:    true,
 			wantAllReady:   false,
 			wantAnyFailed:  false,
 			wantReadySlots: 0,
@@ -1212,6 +1296,7 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) {
 			name:           "parentGen written to spec but echo not yet in status: nudge annotation set",
 			obsGen:         1, // obs behind cr.Generation=2; spec.parentGeneration already set to cr.Generation
 			condStatus:     metav1.ConditionTrue,
+			shouldNudge:    true,
 			wantAllReady:   false,
 			wantAnyFailed:  false,
 			wantReadySlots: 0,
@@ -1259,7 +1344,7 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) {
 			}
 
 			controller := &CommittedResourceController{Client: k8sClient, Scheme: scheme}
-			allReady, anyFailed, readySlots, reason, _, failedSlots, err := controller.checkChildReservationStatus(context.Background(), cr, 1)
+			allReady, anyFailed, readySlots, reason, _, failedSlots, err := controller.checkChildReservationStatus(context.Background(), cr, 1, tt.shouldNudge)
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)
 			}

From 68e28e7b40a8d2d66fa07e54fa32f9c5820d49ec Mon Sep 17 00:00:00 2001
From: Marcel <156897072+mblos@users.noreply.github.com>
Date: Mon, 12 Oct 2026 11:52:05 +0200
Subject: [PATCH 7/9] refactor

Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com>
---
 .../committed_resource_controller.go          | 43 +++++++++----------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go
index 89b6f3ac1..dd28d774a 100644
--- a/internal/scheduling/reservations/commitments/committed_resource_controller.go
+++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go
@@ -166,7 +166,7 @@ func (r *CommittedResourceController) reconcilePending(ctx context.Context, logg
 		if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, msg, true); err != nil {
 			return ctrl.Result{}, err
 		}
-		return ctrl.Result{RequeueAfter: time.Second}, nil
+		return ctrl.Result{RequeueAfter: 3 * time.Second}, nil
 	}
 	logger.Info("committed resource accepted", "generation", cr.Generation, "amount", cr.Spec.Amount.String())
 	return ctrl.Result{}, r.setAccepted(ctx, cr)
@@ -269,7 +269,7 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo
 		if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, msg, true); err != nil {
 			return ctrl.Result{}, err
 		}
-		return ctrl.Result{RequeueAfter: time.Second}, nil
+		return ctrl.Result{RequeueAfter: 3 * time.Second}, nil
 	}
 	logger.Info("committed resource accepted", "generation", cr.Generation, "amount", cr.Spec.Amount.String())
 	return ctrl.Result{}, r.setAccepted(ctx, cr)
@@ -454,30 +454,29 @@ func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Co
 			// parentGen has been written to the Reservation spec (syncReservationMetadata
 			// already ran for this generation) but the Reservation controller hasn't echoed
 			// it to status yet. Poke the Reservation to generate a fresh watch event so the
-			// Reservation controller wakes up and fires the echo. Only nudge once per CR
-			// generation: if the annotation already carries the current generation we already
-			// sent the watch event — skip silently and let the 1s fallback requeue do the
-			// waiting. This bounds the Info log to at most one occurrence per slot per generation.
+			// Reservation controller wakes up and fires the echo.
+			// Use gen+ms as the annotation value so each nudge attempt is a distinct string:
+			// a same-value patch is a no-op on the API server (no watch event generated), and
+			// in a double-stale scenario the first nudge may itself be processed with a stale
+			// object, so periodic retries (every 3s via the fallback requeue) are needed.
 			// shouldNudge is false on the first cycle (CR condition not yet set for this
-			// generation) to give the normal echo path ~1s to complete before intervening.
+			// generation) to give the normal echo path ~3s to complete before intervening.
 			if shouldNudge &&
 				res.Spec.CommittedResourceReservation != nil &&
 				res.Spec.CommittedResourceReservation.ParentGeneration == cr.Generation {
-				currentGen := strconv.FormatInt(cr.Generation, 10)
-				if res.Annotations["cortex.cloud/echo-nudge"] != currentGen {
-					ctrl.LoggerFrom(ctx).Info("nudging stale reservation: parentGen written to spec but echo pending",
-						"slot", res.Name,
-						"crGen", cr.Generation,
-						"obs", obs,
-					)
-					old := res.DeepCopy()
-					if res.Annotations == nil {
-						res.Annotations = map[string]string{}
-					}
-					res.Annotations["cortex.cloud/echo-nudge"] = currentGen
-					if pErr := r.Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(pErr) != nil {
-						ctrl.LoggerFrom(ctx).V(1).Info("failed to nudge stale reservation echo", "slot", res.Name, "err", pErr)
-					}
+				nudgeVal := strconv.FormatInt(cr.Generation, 10) + "-" + strconv.FormatInt(time.Now().UnixMilli(), 10)
+				ctrl.LoggerFrom(ctx).Info("nudging stale reservation: parentGen written to spec but echo pending",
+					"slot", res.Name,
+					"crGen", cr.Generation,
+					"obs", obs,
+				)
+				old := res.DeepCopy()
+				if res.Annotations == nil {
+					res.Annotations = map[string]string{}
+				}
+				res.Annotations["cortex.cloud/echo-nudge"] = nudgeVal
+				if pErr := r.Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(pErr) != nil {
+					ctrl.LoggerFrom(ctx).V(1).Info("failed to nudge stale reservation echo", "slot", res.Name, "err", pErr)
 				}
 			}
 			allReady = false

From 0ca8c3ceca625b905370e1fa92ed99df4c8ebe8e Mon Sep 17 00:00:00 2001
From: Marcel <156897072+mblos@users.noreply.github.com>
Date: Mon, 12 Oct 2026 14:46:43 +0200
Subject: [PATCH 8/9] fix stuck generation

Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com>
---
 .../commitments/committed_resource_controller.go    | 13 ++++++++++++-
 .../reservations/commitments/reservation_manager.go |  5 ++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go
index dd28d774a..0741195c6 100644
--- a/internal/scheduling/reservations/commitments/committed_resource_controller.go
+++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go
@@ -272,7 +272,18 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo
 		return ctrl.Result{RequeueAfter: 3 * time.Second}, nil
 	}
 	logger.Info("committed resource accepted", "generation", cr.Generation, "amount", cr.Spec.Amount.String())
-	return ctrl.Result{}, r.setAccepted(ctx, cr)
+	if err := r.setAccepted(ctx, cr); err != nil {
+		return ctrl.Result{}, err
+	}
+	var latest v1alpha1.CommittedResource
+	if err := r.Get(ctx, client.ObjectKeyFromObject(cr), &latest); err == nil && latest.Generation > cr.Generation {
+		logger.Info("generation advanced during reconcile, requeueing",
+			"processedGen", cr.Generation,
+			"latestGen", latest.Generation,
+		)
+		return ctrl.Result{RequeueAfter: 1 * time.Second}, nil
+	}
+	return ctrl.Result{}, nil
 }
 
 // reconcileCoresHeadroom handles acceptance of CPU core commitments.
diff --git a/internal/scheduling/reservations/commitments/reservation_manager.go b/internal/scheduling/reservations/commitments/reservation_manager.go
index 68eae90b7..cc83851e5 100644
--- a/internal/scheduling/reservations/commitments/reservation_manager.go
+++ b/internal/scheduling/reservations/commitments/reservation_manager.go
@@ -403,7 +403,10 @@ func (m *ReservationManager) syncReservationMetadata(
 		// Apply patch
 		logger.V(1).Info("syncing reservation metadata",
 			"reservation", reservation.Name,
-			"commitmentUUID", state.CommitmentUUID)
+			"commitmentUUID", state.CommitmentUUID,
+			"currentParentGen", reservation.Spec.CommittedResourceReservation.ParentGeneration,
+			"newParentGen", state.ParentGeneration,
+		)
 
 		patch := client.MergeFrom(reservation.DeepCopy())
 

From b234377ce74298135b99dcd0237c005ae74083b3 Mon Sep 17 00:00:00 2001
From: Marcel <156897072+mblos@users.noreply.github.com>
Date: Mon, 12 Oct 2026 14:48:45 +0200
Subject: [PATCH 9/9] .

Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com>
---
 .../reservations/commitments/committed_resource_controller.go  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go
index 0741195c6..f8703d734 100644
--- a/internal/scheduling/reservations/commitments/committed_resource_controller.go
+++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go
@@ -57,7 +57,8 @@ func (r *CommittedResourceController) Reconcile(ctx context.Context, req ctrl.Re
 		"committedResource", req.Name,
 	)
 
-	// Debug: timing anchor for CR reconcile relative to Phase 6 and Reservation echo.
+	// Timing anchor: shows which generation the cache served when this reconcile fired.
+	// Missing for a specific generation means the CR controller never dequeued that gen.
 	logger.V(1).Info("reconcile entry", "rv", cr.ResourceVersion, "gen", cr.Generation)
 
 	if !cr.DeletionTimestamp.IsZero() {