diff --git a/api/v1/eviction_types.go b/api/v1/eviction_types.go index b96ad007..5f6039f2 100644 --- a/api/v1/eviction_types.go +++ b/api/v1/eviction_types.go @@ -53,6 +53,13 @@ const ( // ConditionTypeHypervisorDisabled is the type of condition for hypervisor disabled status ConditionTypeHypervisorDisabled = "HypervisorDisabled" + // ConditionTypeIncomingMigrationsSettled is set on the Hypervisor CR by the + // maintenance controller. True means no live-migrations or evacuations are + // currently in flight targeting this host. False/Unknown/Missing means the + // controller has observed outstanding incoming migrations (or hasn't checked yet) + // and the host must not be declared evicted. + ConditionTypeIncomingMigrationsSettled = "IncomingMigrationsSettled" + // ConditionTypeHaEnabled is the type of condition for signalling if HA is enabled / disabled for the hypervisor ConditionTypeHaEnabled = "HaEnabled" @@ -70,6 +77,15 @@ const ( // ConditionReasonSucceeded means the eviction has succeeded ConditionReasonSucceeded string = "Succeeded" + + // ConditionReasonSettled means all incoming migrations have reached a terminal state + ConditionReasonSettled string = "Settled" + + // ConditionReasonAborting means the controller is aborting incoming migrations + ConditionReasonAborting string = "AbortingIncomingMigrations" + + // ConditionReasonWaiting means the controller is waiting for non-abortable incoming migrations + ConditionReasonWaiting string = "WaitingForIncomingMigrations" ) // EvictionStatus defines the observed state of Eviction diff --git a/internal/controller/hypervisor_maintenance_controller.go b/internal/controller/hypervisor_maintenance_controller.go index 7cc482f9..fee246db 100644 --- a/internal/controller/hypervisor_maintenance_controller.go +++ b/internal/controller/hypervisor_maintenance_controller.go @@ -24,6 +24,7 @@ import ( "context" "errors" "fmt" + "time" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,6 +45,10 @@ import ( const ( HypervisorMaintenanceControllerName = "HypervisorMaintenance" + + // settleRequeueInterval is the interval at which the controller re-checks + // incoming migrations when they haven't settled yet. + settleRequeueInterval = 10 * time.Second ) type HypervisorMaintenanceController struct { @@ -74,19 +79,24 @@ func (hec *HypervisorMaintenanceController) Reconcile(ctx context.Context, req c if err := hec.reconcileComputeService(ctx, hv, statusCfg); err != nil { retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) applyErr := hec.Status().Apply(ctx, apiv1.Hypervisor(hv.Name).WithStatus(statusCfg), k8sclient.ForceOwnership, k8sclient.FieldOwner(HypervisorMaintenanceControllerName)) return ctrl.Result{}, errors.Join(err, applyErr) } - if err := hec.reconcileEviction(ctx, hv, statusCfg); err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, hec.Status().Apply(ctx, + requeue, err := hec.reconcileEviction(ctx, hv, statusCfg) + applyErr := hec.Status().Apply(ctx, apiv1.Hypervisor(hv.Name).WithStatus(statusCfg), k8sclient.ForceOwnership, k8sclient.FieldOwner(HypervisorMaintenanceControllerName)) + if err != nil || applyErr != nil { + return ctrl.Result{}, errors.Join(err, applyErr) + } + if requeue { + return ctrl.Result{RequeueAfter: settleRequeueInterval}, nil + } + return ctrl.Result{}, nil } // retainStatusCondition keeps an existing condition of the given type in the @@ -175,9 +185,10 @@ func (hec *HypervisorMaintenanceController) reconcileComputeService(ctx context. } // reconcileEviction creates/deletes the Eviction CR and sets the ConditionTypeEvicting -// condition and Evicted scalar on statusCfg. When eviction should be removed, the -// condition entry is filtered out so SSA prunes it. -func (hec *HypervisorMaintenanceController) reconcileEviction(ctx context.Context, hv *kvmv1.Hypervisor, statusCfg *apiv1.HypervisorStatusApplyConfiguration) error { +// and ConditionTypeIncomingMigrationsSettled conditions plus the Evicted scalar on +// statusCfg. When eviction should be removed, the condition entries are filtered out +// so SSA prunes them. +func (hec *HypervisorMaintenanceController) reconcileEviction(ctx context.Context, hv *kvmv1.Hypervisor, statusCfg *apiv1.HypervisorStatusApplyConfiguration) (requeue bool, err error) { eviction := &kvmv1.Eviction{ Name: hv.Name, } @@ -185,35 +196,102 @@ func (hec *HypervisorMaintenanceController) reconcileEviction(ctx context.Contex switch hv.Spec.Maintenance { case kvmv1.MaintenanceUnset: // Avoid deleting the eviction over and over. - if !hv.Status.Evicted && meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting) == nil { - return nil + if !hv.Status.Evicted && + meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting) == nil && + meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) == nil { + return false, nil } if err := k8sclient.IgnoreNotFound(hec.Delete(ctx, eviction)); err != nil { - return err + return false, err } - // ConditionTypeEvicting is intentionally absent from statusCfg — SSA - // will prune it from this field manager's managed fields on Apply. + // ConditionTypeEvicting and ConditionTypeIncomingMigrationsSettled are + // intentionally absent from statusCfg — SSA will prune them from this + // field manager's managed fields on Apply. statusCfg.WithEvicted(false) case kvmv1.MaintenanceManual, kvmv1.MaintenanceAuto, kvmv1.MaintenanceTermination: - // In case of "ha", the host gets emptied from the HA service. - // Seed the existing evicting condition so SSA does not prune it, - // regardless of whether we take the short-circuit below. - if cond := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting); cond != nil { - statusCfg.WithConditions(utils.ConditionFromStatus(*cond)) - if cond.Reason == kvmv1.ConditionReasonSucceeded { - // We are done here, no need to look at the eviction any more. - return nil + evictingCondition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + migrationsCondition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + if evictingCondition != nil && + evictingCondition.Status == metav1.ConditionFalse && + evictingCondition.Reason == kvmv1.ConditionReasonSucceeded { + if hv.Status.NumInstances > 0 { + // Instance appeared after eviction completed. Delete the Eviction CR so + // ensureEviction recreates a fresh one. + log := logger.FromContext(ctx) + log.Info("Eviction reported succeeded but instances remain on host; re-entering drain", + "numInstances", hv.Status.NumInstances) + if delErr := k8sclient.IgnoreNotFound(hec.Delete(ctx, eviction)); delErr != nil { + return false, delErr + } + statusCfg.WithEvicted(false) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeEvicting). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonRunning). + WithMessage(fmt.Sprintf("Re-entering drain: %d instance(s) still on host", hv.Status.NumInstances))) + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + return false, nil + } + + if hv.Status.Evicted && migrationsCondition != nil && + migrationsCondition.Status == metav1.ConditionTrue && + migrationsCondition.Reason == kvmv1.ConditionReasonSettled { + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + return false, nil } } + // Gate: ensure all incoming migrations are settled before proceeding. + settled, err := hec.settleIncomingMigrations(ctx, hv, statusCfg) + if err != nil { + statusCfg.WithEvicted(false) + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + return false, err + } + if !settled { + // Cannot proceed with eviction while incoming migrations are outstanding. + // Clear the terminal scalar and seed the existing evicting condition so + // SSA does not leave the hypervisor marked as evicted or prune it. + statusCfg.WithEvicted(false) + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + return true, nil + } + + if evictingCondition != nil && evictingCondition.Reason == kvmv1.ConditionReasonSucceeded { + // The host was not previously known to be settled, but the current Nova + // check established it. Preserve the completed eviction state. + retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeEvicting) + return false, nil + } + status, err := hec.ensureEviction(ctx, eviction, hv) if err != nil { - return err + return false, err } var reason, message string if status == metav1.ConditionFalse { + if hv.Status.NumInstances > 0 { + // Eviction CR says done but instances exist (race with in-flight migration). + // Delete the Eviction CR and restart drain. + log := logger.FromContext(ctx) + log.Info("Eviction finished but instances remain on host; restarting drain", + "numInstances", hv.Status.NumInstances) + if delErr := k8sclient.IgnoreNotFound(hec.Delete(ctx, eviction)); delErr != nil { + return false, delErr + } + statusCfg.WithEvicted(false) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeEvicting). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonRunning). + WithMessage(fmt.Sprintf("Restarting drain: %d instance(s) still on host", hv.Status.NumInstances))) + return false, nil + } message = "Evicted" reason = kvmv1.ConditionReasonSucceeded statusCfg.WithEvicted(true) @@ -231,7 +309,66 @@ func (hec *HypervisorMaintenanceController) reconcileEviction(ctx context.Contex WithMessage(message)) } - return nil + return false, nil +} + +// settleIncomingMigrations checks for in-flight migrations targeting this host, +// aborts those that can be aborted, and reports whether the host is settled. +// It sets the IncomingMigrationsSettled condition on statusCfg. +// Returns true if settled (no incoming migrations), false otherwise. +func (hec *HypervisorMaintenanceController) settleIncomingMigrations(ctx context.Context, hv *kvmv1.Hypervisor, statusCfg *apiv1.HypervisorStatusApplyConfiguration) (bool, error) { + log := logger.FromContext(ctx) + + aborted, waiting, err := openstack.SettleIncomingMigrations(ctx, hec.computeClient, hv.Name) + if err != nil { + err = fmt.Errorf("settling incoming migrations for %s: %w", hv.Name, err) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeIncomingMigrationsSettled). + WithStatus(metav1.ConditionUnknown). + WithReason(kvmv1.ConditionReasonFailed). + WithMessage(err.Error())) + return false, err + } + + if len(waiting) > 0 { + instanceUUIDs := make([]string, len(waiting)) + for i, m := range waiting { + instanceUUIDs[i] = m.InstanceUUID + } + log.Info("Waiting for non-abortable incoming migrations", "host", hv.Name, "instances", instanceUUIDs) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeIncomingMigrationsSettled). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonWaiting). + WithMessage(fmt.Sprintf("Waiting for %d non-abortable incoming migration(s) to complete", len(waiting)))) + return false, nil + } + + if len(aborted) > 0 { + instanceUUIDs := make([]string, len(aborted)) + for i, m := range aborted { + instanceUUIDs[i] = m.InstanceUUID + } + log.Info("Aborted incoming migrations", "host", hv.Name, "instances", instanceUUIDs) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeIncomingMigrationsSettled). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonAborting). + WithMessage(fmt.Sprintf("Aborted %d incoming migration(s); verifying on next reconcile", len(aborted)))) + return false, nil + } + + // No incoming migrations — settled. + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeIncomingMigrationsSettled). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonSettled). + WithMessage("No incoming migrations targeting this host")) + return true, nil } func (hec *HypervisorMaintenanceController) ensureEviction(ctx context.Context, eviction *kvmv1.Eviction, hypervisor *kvmv1.Hypervisor) (metav1.ConditionStatus, error) { diff --git a/internal/controller/hypervisor_maintenance_controller_test.go b/internal/controller/hypervisor_maintenance_controller_test.go index 27990d44..af7d5e75 100644 --- a/internal/controller/hypervisor_maintenance_controller_test.go +++ b/internal/controller/hypervisor_maintenance_controller_test.go @@ -20,6 +20,7 @@ package controller import ( "fmt" "net/http" + "time" "github.com/gophercloud/gophercloud/v2/testhelper" "github.com/gophercloud/gophercloud/v2/testhelper/client" @@ -28,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + applymetav1 "k8s.io/client-go/applyconfigurations/meta/v1" ctrl "sigs.k8s.io/controller-runtime" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -43,6 +45,10 @@ var _ = Describe("HypervisorMaintenanceController", func() { hypervisorName = types.NamespacedName{Name: "hv-test"} expectReconcileErr bool reconcileErr error + migrationsResponse string // mutable: tests can set this to control /os-migrations responses + deletedMigrations []string + migrationRequests int + migrationsStatus int ) const ( @@ -88,6 +94,35 @@ var _ = Describe("HypervisorMaintenanceController", func() { expectReconcileErr = false reconcileErr = nil + // Default: no incoming migrations + migrationsResponse = `{"migrations": []}` + deletedMigrations = nil + migrationRequests = 0 + migrationsStatus = http.StatusOK + + fakeServer.Mux.HandleFunc("GET /os-migrations", func(w http.ResponseWriter, r *http.Request) { + migrationRequests++ + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(migrationsStatus) + if migrationsStatus != http.StatusOK { + fmt.Fprint(w, `{"error": "nova unavailable"}`) + return + } + mt := r.URL.Query().Get("migration_type") + // Return the configured migrations for the default query; + // evacuation queries return empty by default. + if mt == "evacuation" { + fmt.Fprint(w, `{"migrations": []}`) + } else { + fmt.Fprint(w, migrationsResponse) + } + }) + + fakeServer.Mux.HandleFunc("DELETE /servers/", func(w http.ResponseWriter, r *http.Request) { + deletedMigrations = append(deletedMigrations, r.URL.Path) + w.WriteHeader(http.StatusAccepted) + }) + By("Creating the HypervisorMaintenanceController") controller = &HypervisorMaintenanceController{ Client: k8sClient, @@ -186,6 +221,12 @@ var _ = Describe("HypervisorMaintenanceController", func() { Reason: kvmv1.ConditionReasonRunning, Message: "Evicting", }) + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeIncomingMigrationsSettled, + Status: metav1.ConditionFalse, + Reason: kvmv1.ConditionReasonWaiting, + Message: "Waiting for incoming migrations", + }) Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) hypervisor.Spec.Maintenance = kvmv1.MaintenanceAuto @@ -208,6 +249,9 @@ var _ = Describe("HypervisorMaintenanceController", func() { evictingCondition := meta.FindStatusCondition(updated.Status.Conditions, kvmv1.ConditionTypeEvicting) Expect(evictingCondition).NotTo(BeNil()) Expect(evictingCondition.Status).To(Equal(metav1.ConditionTrue)) + migrationsCondition := meta.FindStatusCondition(updated.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + Expect(migrationsCondition).NotTo(BeNil()) + Expect(migrationsCondition.Status).To(Equal(metav1.ConditionFalse)) eviction := &kvmv1.Eviction{} err := k8sClient.Get(ctx, hypervisorName, eviction) @@ -312,6 +356,42 @@ var _ = Describe("HypervisorMaintenanceController", func() { }) }) // Spec.Maintenance="" + Context("Spec.Maintenance=\"\" with only stale IncomingMigrationsSettled condition", func() { + BeforeEach(func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + + // First, simulate that the controller previously owned the condition + // by applying it via SSA with the same field manager. + statusCfg := applyv1.HypervisorStatus(). + WithEvicted(false). + WithConditions(applymetav1.Condition(). + WithType(kvmv1.ConditionTypeIncomingMigrationsSettled). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonWaiting). + WithMessage("incoming migrations still pending"). + WithLastTransitionTime(metav1.Now())) + Expect(k8sClient.Status().Apply(ctx, + applyv1.Hypervisor(hypervisor.Name).WithStatus(statusCfg), + k8sclient.ForceOwnership, + k8sclient.FieldOwner(HypervisorMaintenanceControllerName), + )).To(Succeed()) + + // Now cancel maintenance. + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Spec.Maintenance = "" + Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed()) + expectedBody := `{"status": "enabled", "forced_down": false}` + mockServiceUpdate(expectedBody) + }) + + It("should remove the stale IncomingMigrationsSettled condition", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled)).To(BeNil()) + }) + }) + Context("Spec.Maintenance=\"ha\"", func() { BeforeEach(func(ctx SpecContext) { hypervisor := &kvmv1.Hypervisor{} @@ -556,6 +636,338 @@ var _ = Describe("HypervisorMaintenanceController", func() { Expect(hypervisor.Status.ServiceID).To(BeEmpty()) }) }) + + Context("Incoming migrations settling", func() { + BeforeEach(func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.ServiceID = "1234" + meta.SetStatusCondition(&hypervisor.Status.Conditions, + metav1.Condition{ + Type: kvmv1.ConditionTypeOnboarding, + Status: metav1.ConditionFalse, + Reason: metav1.StatusSuccess, + Message: "Onboarded", + }, + ) + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + + // Re-read to get fresh resourceVersion, then update spec + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Spec.Maintenance = "auto" + Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed()) + + // Permissive service mock: accept any enable/disable call + fakeServer.Mux.HandleFunc("PUT /os-services/1234", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, ServiceEnabledResponse) + }) + }) + + Context("when there are no incoming migrations", func() { + It("should set IncomingMigrationsSettled=True and proceed to create eviction", func(ctx SpecContext) { + Expect(migrationRequests).To(Equal(1)) + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(meta.IsStatusConditionTrue(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled)).To(BeTrue()) + // Eviction should be created + eviction := &kvmv1.Eviction{} + Expect(k8sClient.Get(ctx, hypervisorName, eviction)).To(Succeed()) + }) + }) + + Context("when Nova cannot list incoming migrations", func() { + BeforeEach(func(ctx SpecContext) { + expectReconcileErr = true + migrationsStatus = http.StatusServiceUnavailable + + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.Evicted = true + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonRunning, + Message: "Evicting", + }) + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeIncomingMigrationsSettled, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonSettled, + Message: "No incoming migrations targeting this host", + }) + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + }) + + It("reports that migration settlement is unknown", func(ctx SpecContext) { + Expect(reconcileErr).To(MatchError(ContainSubstring("settling incoming migrations"))) + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + condition := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + Expect(condition).NotTo(BeNil()) + Expect(condition.Status).To(Equal(metav1.ConditionUnknown)) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonFailed)) + Expect(condition.Message).To(ContainSubstring("nova unavailable")) + Expect(hypervisor.Status.Evicted).To(BeFalse()) + evictingCondition := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeEvicting) + Expect(evictingCondition).NotTo(BeNil()) + Expect(evictingCondition.Status).To(Equal(metav1.ConditionTrue)) + Expect(evictingCondition.Reason).To(Equal(kvmv1.ConditionReasonRunning)) + }) + + It("does not create an eviction", func(ctx SpecContext) { + eviction := &kvmv1.Eviction{} + err := k8sClient.Get(ctx, hypervisorName, eviction) + Expect(err).To(HaveOccurred()) + Expect(k8sclient.IgnoreNotFound(err)).To(Succeed()) + }) + }) + + Context("when the host is conclusively evicted", func() { + BeforeEach(func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.Evicted = true + hypervisor.Status.NumInstances = 0 + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionFalse, + Reason: kvmv1.ConditionReasonSucceeded, + Message: "Evicted", + }) + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeIncomingMigrationsSettled, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonSettled, + Message: "No incoming migrations targeting this host", + }) + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + }) + + It("does not query Nova again", func(ctx SpecContext) { + Expect(migrationRequests).To(BeZero()) + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Evicted).To(BeTrue()) + Expect(meta.IsStatusConditionTrue(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled)).To(BeTrue()) + }) + }) + + Context("when there is a running incoming migration", func() { + BeforeEach(func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.Evicted = true + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + + migrationsResponse = `{"migrations": [ + { + "id": 42, + "uuid": "mig-uuid-1", + "instance_uuid": "inst-uuid-1", + "status": "running", + "source_compute": "node003", + "dest_compute": "hv-test", + "migration_type": "live-migration" + } + ]}` + }) + + It("should set IncomingMigrationsSettled=False with Aborting reason", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + cond := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(kvmv1.ConditionReasonAborting)) + }) + + It("should clear a previously reported evicted status", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Evicted).To(BeFalse()) + }) + + It("should issue a DELETE to abort the migration", func(_ SpecContext) { + Expect(deletedMigrations).To(HaveLen(1)) + Expect(deletedMigrations[0]).To(ContainSubstring("inst-uuid-1")) + Expect(deletedMigrations[0]).To(ContainSubstring("42")) + }) + + It("should not create an eviction resource", func(ctx SpecContext) { + eviction := &kvmv1.Eviction{} + err := k8sClient.Get(ctx, hypervisorName, eviction) + Expect(err).To(HaveOccurred()) + Expect(k8sclient.IgnoreNotFound(err)).To(Succeed()) + }) + + It("should requeue after settleRequeueInterval", func(ctx SpecContext) { + req := ctrl.Request{NamespacedName: hypervisorName} + result, err := controller.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(Equal(10 * time.Second)) + }) + }) + + Context("when there is a post-migrating incoming migration", func() { + BeforeEach(func(_ SpecContext) { + migrationsResponse = `{"migrations": [ + { + "id": 99, + "uuid": "mig-uuid-2", + "instance_uuid": "inst-uuid-2", + "status": "post-migrating", + "source_compute": "node003", + "dest_compute": "hv-test", + "migration_type": "live-migration" + } + ]}` + }) + + It("should set IncomingMigrationsSettled=False with Waiting reason", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + cond := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(kvmv1.ConditionReasonWaiting)) + }) + + It("should not issue any DELETE", func(_ SpecContext) { + Expect(deletedMigrations).To(BeEmpty()) + }) + + It("should not create an eviction resource", func(ctx SpecContext) { + eviction := &kvmv1.Eviction{} + err := k8sClient.Get(ctx, hypervisorName, eviction) + Expect(err).To(HaveOccurred()) + Expect(k8sclient.IgnoreNotFound(err)).To(Succeed()) + }) + + It("should requeue after settleRequeueInterval", func(ctx SpecContext) { + req := ctrl.Request{NamespacedName: hypervisorName} + result, err := controller.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(Equal(10 * time.Second)) + }) + }) + + Context("when eviction succeeded but instances remain (incident regression)", func() { + BeforeEach(func(ctx SpecContext) { + // Simulate: eviction CR exists and reports Succeeded + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + + eviction := &kvmv1.Eviction{ + ObjectMeta: metav1.ObjectMeta{Name: hypervisorName.Name}, + Spec: kvmv1.EvictionSpec{ + Hypervisor: hypervisorName.Name, + Reason: "test", + }, + } + Expect(controllerutil.SetControllerReference(hypervisor, eviction, controller.Scheme)).To(Succeed()) + Expect(k8sClient.Create(ctx, eviction)).To(Succeed()) + + meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionFalse, + Message: "done", + Reason: kvmv1.ConditionReasonSucceeded, + }) + Expect(k8sClient.Status().Update(ctx, eviction)).To(Succeed()) + + // Set NumInstances=1 on the Hypervisor (simulating late-arriving instance) + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.NumInstances = 1 + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionFalse, + Reason: kvmv1.ConditionReasonSucceeded, + Message: "Evicted", + }) + hypervisor.Status.Evicted = true + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + }) + + It("should flip Evicted back to false", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Evicted).To(BeFalse()) + }) + + It("should set Evicting back to Running", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + cond := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeEvicting) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(kvmv1.ConditionReasonRunning)) + }) + + It("should delete the eviction CR to re-enter drain", func(ctx SpecContext) { + eviction := &kvmv1.Eviction{} + err := k8sClient.Get(ctx, hypervisorName, eviction) + Expect(err).To(HaveOccurred()) + Expect(k8sclient.IgnoreNotFound(err)).To(Succeed()) + }) + }) + + Context("when eviction just finished but instances remain (no pre-existing Succeeded condition)", func() { + BeforeEach(func(ctx SpecContext) { + // Simulate: eviction CR exists and reports finished, but the + // Hypervisor does NOT have Evicting=Succeeded yet (first reconcile + // after the Eviction CR transitioned). + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + + eviction := &kvmv1.Eviction{ + ObjectMeta: metav1.ObjectMeta{Name: hypervisorName.Name}, + Spec: kvmv1.EvictionSpec{ + Hypervisor: hypervisorName.Name, + Reason: "test", + }, + } + Expect(controllerutil.SetControllerReference(hypervisor, eviction, controller.Scheme)).To(Succeed()) + Expect(k8sClient.Create(ctx, eviction)).To(Succeed()) + + meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionFalse, + Message: "done", + Reason: kvmv1.ConditionReasonSucceeded, + }) + Expect(k8sClient.Status().Update(ctx, eviction)).To(Succeed()) + + // Set NumInstances=1 but leave Evicting condition as Running (not Succeeded) + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + hypervisor.Status.NumInstances = 1 + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonRunning, + Message: "Evicting", + }) + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + }) + + It("should set Evicting to True/Running and not declare Evicted", func(ctx SpecContext) { + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Evicted).To(BeFalse()) + cond := meta.FindStatusCondition(hypervisor.Status.Conditions, kvmv1.ConditionTypeEvicting) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(kvmv1.ConditionReasonRunning)) + }) + + It("should delete the eviction CR to restart drain", func(ctx SpecContext) { + eviction := &kvmv1.Eviction{} + err := k8sClient.Get(ctx, hypervisorName, eviction) + Expect(err).To(HaveOccurred()) + Expect(k8sclient.IgnoreNotFound(err)).To(Succeed()) + }) + }) + }) }) var _ = Describe("retainStatusCondition", func() { diff --git a/internal/openstack/migrations.go b/internal/openstack/migrations.go new file mode 100644 index 00000000..9a92ee20 --- /dev/null +++ b/internal/openstack/migrations.go @@ -0,0 +1,173 @@ +/* +SPDX-FileCopyrightText: Copyright 2025 SAP SE or an SAP affiliate company and cobaltcore-dev contributors +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openstack + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/gophercloud/gophercloud/v2" +) + +// IncomingMigrationsLookbackWindow defines how far back we query Nova for +// migrations. Nova's live-migration monitor updates Migration.updated_at every +// ~5 seconds on both libvirt/kvm and libvirt/cloud-hypervisor backends, so any +// in-flight migration is guaranteed to appear in this window. +const IncomingMigrationsLookbackWindow = 6 * time.Hour + +// MigrationInfo holds the subset of Nova migration fields relevant to settling +// incoming migrations on a host being drained. +type MigrationInfo struct { + ID int `json:"id"` + UUID string `json:"uuid"` + InstanceUUID string `json:"instance_uuid"` + Status string `json:"status"` + SourceCompute string `json:"source_compute"` + DestCompute string `json:"dest_compute"` + MigrationType string `json:"migration_type"` +} + +// migrationsResponse mirrors Nova's GET /os-migrations JSON envelope. +type migrationsResponse struct { + Migrations []MigrationInfo `json:"migrations"` +} + +// terminalStatuses are migration statuses that no longer represent an in-flight +// migration. Derived from nova/db/main/api.py migration_get_in_progress_by_host_and_node. +var terminalStatuses = map[string]bool{ + "confirmed": true, + "reverted": true, + "error": true, + "failed": true, + "completed": true, + "cancelled": true, + "done": true, +} + +// abortableStatuses are migration statuses where a DELETE (abort) is supported. +// queued and preparing require microversion >= 2.65; running is always abortable. +// The operator uses microversion 2.90 which covers all three. +var abortableStatuses = map[string]bool{ + "queued": true, + "preparing": true, + "running": true, +} + +// ListActiveIncomingMigrations queries Nova for migrations that target the +// given host and are still in a non-terminal state. It issues a single call +// with a changes-since window of IncomingMigrationsLookbackWindow, then +// filters client-side for dest_compute==host and non-terminal status. +func ListActiveIncomingMigrations(ctx context.Context, sc *gophercloud.ServiceClient, host string) ([]MigrationInfo, error) { + changesSince := time.Now().UTC().Add(-IncomingMigrationsLookbackWindow).Format(time.RFC3339) + + migrations, err := listMigrationsForHost(ctx, sc, host, changesSince) + if err != nil { + return nil, fmt.Errorf("listing migrations for host %s: %w", host, err) + } + + var result []MigrationInfo + for _, m := range migrations { + if m.DestCompute != host { + continue + } + if terminalStatuses[m.Status] { + continue + } + result = append(result, m) + } + + return result, nil +} + +// AbortMigration issues DELETE /servers/{instanceUUID}/migrations/{migrationID} +// to abort a live-migration. Treats HTTP 404 and 409 as non-fatal (the +// migration already transitioned past an abortable state or was deleted). +func AbortMigration(ctx context.Context, sc *gophercloud.ServiceClient, instanceUUID string, migrationID int) error { + deleteURL := sc.ServiceURL("servers", instanceUUID, "migrations", strconv.Itoa(migrationID)) + + resp, err := sc.Delete(ctx, deleteURL, &gophercloud.RequestOpts{ + OkCodes: []int{http.StatusAccepted, http.StatusNoContent}, + }) + if err != nil { + if gophercloud.ResponseCodeIs(err, http.StatusNotFound) || gophercloud.ResponseCodeIs(err, http.StatusConflict) { + // Migration already completed/cancelled or not found — non-fatal. + return nil + } + return fmt.Errorf("aborting migration %d for instance %s: %w", migrationID, instanceUUID, err) + } + // gophercloud closes the response body when JSONResponse is nil, + // but close defensively if it wasn't. + if resp != nil && resp.Body != nil { + resp.Body.Close() + } + return nil +} + +// SettleIncomingMigrations combines listing and aborting: it lists all active +// incoming migrations for the host, aborts those in abortable states, and +// returns both the aborted and the waiting (post-migrating) sets. +// +// The caller should treat: +// - len(aborted) > 0 as "just issued aborts, recheck on next reconcile" +// - len(waiting) > 0 as "cannot abort, must wait for completion" +// - both empty as "no incoming migrations; host is settled" +func SettleIncomingMigrations(ctx context.Context, sc *gophercloud.ServiceClient, host string) (aborted, waiting []MigrationInfo, err error) { + active, err := ListActiveIncomingMigrations(ctx, sc, host) + if err != nil { + return nil, nil, err + } + + for _, m := range active { + if abortableStatuses[m.Status] && m.MigrationType == "live-migration" { + if abortErr := AbortMigration(ctx, sc, m.InstanceUUID, m.ID); abortErr != nil { + return aborted, waiting, abortErr + } + aborted = append(aborted, m) + } else { + // Non-abortable: either post-migrating, or an evacuation (which Nova + // does not support aborting). Must wait for completion. + waiting = append(waiting, m) + } + } + + return aborted, waiting, nil +} + +// listMigrationsForHost queries GET /os-migrations with host and changes-since filters. +func listMigrationsForHost(ctx context.Context, sc *gophercloud.ServiceClient, host, changesSince string) ([]MigrationInfo, error) { + query := url.Values{} + query.Set("host", host) + query.Set("changes-since", changesSince) + + requestURL := sc.ServiceURL("os-migrations") + "?" + query.Encode() + + var parsed migrationsResponse + //nolint:bodyclose // gophercloud closes the body when JSONResponse is non-nil + _, err := sc.Get(ctx, requestURL, &parsed, &gophercloud.RequestOpts{ + OkCodes: []int{http.StatusOK}, + }) + if err != nil { + return nil, err + } + + return parsed.Migrations, nil +} diff --git a/internal/openstack/migrations_test.go b/internal/openstack/migrations_test.go new file mode 100644 index 00000000..d22b916c --- /dev/null +++ b/internal/openstack/migrations_test.go @@ -0,0 +1,618 @@ +/* +SPDX-FileCopyrightText: Copyright 2025 SAP SE or an SAP affiliate company and cobaltcore-dev contributors +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openstack + +import ( + "context" + "fmt" + "net/http" + + "github.com/gophercloud/gophercloud/v2/testhelper" + "github.com/gophercloud/gophercloud/v2/testhelper/client" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Migrations", func() { + var ( + fakeServer testhelper.FakeServer + ctx context.Context + ) + + BeforeEach(func() { + fakeServer = testhelper.SetupHTTP() + ctx = context.Background() + }) + + AfterEach(func() { + fakeServer.Teardown() + }) + + // migrationsHandler returns a handler that responds to GET /os-migrations. + migrationsHandler := func(data string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, data) + } + } + + emptyMigrations := `{"migrations": []}` + + Describe("ListActiveIncomingMigrations", func() { + Context("when there are no migrations", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(emptyMigrations)) + }) + + It("should return an empty list", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(BeEmpty()) + }) + }) + + Context("when there is a running migration targeting the host", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 42, + "uuid": "mig-uuid-1", + "instance_uuid": "inst-uuid-1", + "status": "running", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should return the migration", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(HaveLen(1)) + Expect(migrations[0].ID).To(Equal(42)) + Expect(migrations[0].InstanceUUID).To(Equal("inst-uuid-1")) + Expect(migrations[0].Status).To(Equal("running")) + Expect(migrations[0].DestCompute).To(Equal("node009-bb549")) + }) + }) + + Context("when there is a post-migrating migration targeting the host", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 99, + "uuid": "mig-uuid-2", + "instance_uuid": "inst-uuid-2", + "status": "post-migrating", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should return the migration as active incoming", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(HaveLen(1)) + Expect(migrations[0].Status).To(Equal("post-migrating")) + }) + }) + + Context("when there is an outbound migration (source_compute == host)", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 50, + "uuid": "mig-uuid-3", + "instance_uuid": "inst-uuid-3", + "status": "running", + "source_compute": "node009-bb549", + "dest_compute": "node005-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should ignore outbound migrations", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(BeEmpty()) + }) + }) + + Context("when there is an incoming evacuation", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 77, + "uuid": "mig-uuid-4", + "instance_uuid": "inst-uuid-4", + "status": "running", + "source_compute": "node001-bb549", + "dest_compute": "node009-bb549", + "migration_type": "evacuation" + } + ]}`)) + }) + + It("should include incoming evacuations", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(HaveLen(1)) + Expect(migrations[0].MigrationType).To(Equal("evacuation")) + Expect(migrations[0].DestCompute).To(Equal("node009-bb549")) + }) + }) + + Context("when migrations have terminal statuses", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 10, + "uuid": "mig-t1", + "instance_uuid": "inst-t1", + "status": "completed", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 11, + "uuid": "mig-t2", + "instance_uuid": "inst-t2", + "status": "cancelled", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 12, + "uuid": "mig-t3", + "instance_uuid": "inst-t3", + "status": "error", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 13, + "uuid": "mig-t4", + "instance_uuid": "inst-t4", + "status": "failed", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 14, + "uuid": "mig-t5", + "instance_uuid": "inst-t5", + "status": "confirmed", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 15, + "uuid": "mig-t6", + "instance_uuid": "inst-t6", + "status": "reverted", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 16, + "uuid": "mig-t7", + "instance_uuid": "inst-t7", + "status": "done", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should filter out all terminal statuses", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(BeEmpty()) + }) + }) + + Context("when Nova returns a server error", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"error": "Internal Server Error"}`) + }) + }) + + It("should propagate the error", func() { + sc := client.ServiceClient(fakeServer) + _, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).To(HaveOccurred()) + }) + }) + + Context("with a mixed set of migrations", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 1, + "uuid": "mig-1", + "instance_uuid": "inst-1", + "status": "running", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 2, + "uuid": "mig-2", + "instance_uuid": "inst-2", + "status": "running", + "source_compute": "node007-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 3, + "uuid": "mig-3", + "instance_uuid": "inst-3", + "status": "post-migrating", + "source_compute": "node002-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 4, + "uuid": "mig-4", + "instance_uuid": "inst-4", + "status": "running", + "source_compute": "node009-bb549", + "dest_compute": "node005-bb549", + "migration_type": "live-migration" + }, + { + "id": 5, + "uuid": "mig-5", + "instance_uuid": "inst-5", + "status": "completed", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should return only active incoming migrations", func() { + sc := client.ServiceClient(fakeServer) + migrations, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(migrations).To(HaveLen(3)) + ids := []int{migrations[0].ID, migrations[1].ID, migrations[2].ID} + Expect(ids).To(ConsistOf(1, 2, 3)) + }) + }) + + Context("verifies query parameters", func() { + var receivedQueries []string + + BeforeEach(func() { + receivedQueries = nil + fakeServer.Mux.HandleFunc("GET /os-migrations", func(w http.ResponseWriter, r *http.Request) { + receivedQueries = append(receivedQueries, r.URL.RawQuery) + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, `{"migrations": []}`) + }) + }) + + It("should send host and changes-since parameters", func() { + sc := client.ServiceClient(fakeServer) + _, err := ListActiveIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + // Single call without migration_type filter + Expect(receivedQueries).To(HaveLen(1)) + Expect(receivedQueries[0]).To(ContainSubstring("host=node009-bb549")) + Expect(receivedQueries[0]).To(ContainSubstring("changes-since=")) + Expect(receivedQueries[0]).NotTo(ContainSubstring("migration_type")) + }) + }) + }) + + Describe("AbortMigration", func() { + Context("when abort succeeds", func() { + var deleteCalled bool + + BeforeEach(func() { + deleteCalled = false + fakeServer.Mux.HandleFunc("DELETE /servers/inst-uuid-1/migrations/42", func(w http.ResponseWriter, r *http.Request) { + deleteCalled = true + w.WriteHeader(http.StatusAccepted) + }) + }) + + It("should issue DELETE and return nil", func() { + sc := client.ServiceClient(fakeServer) + err := AbortMigration(ctx, sc, "inst-uuid-1", 42) + Expect(err).NotTo(HaveOccurred()) + Expect(deleteCalled).To(BeTrue()) + }) + }) + + Context("when Nova returns 404 (migration already gone)", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("DELETE /servers/inst-uuid-1/migrations/42", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, `{"itemNotFound": {"message": "Migration not found", "code": 404}}`) + }) + }) + + It("should treat as non-fatal", func() { + sc := client.ServiceClient(fakeServer) + err := AbortMigration(ctx, sc, "inst-uuid-1", 42) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when Nova returns 409 (migration not abortable)", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("DELETE /servers/inst-uuid-1/migrations/42", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusConflict) + fmt.Fprint(w, `{"conflictingRequest": {"message": "Migration status is post-migrating", "code": 409}}`) + }) + }) + + It("should treat as non-fatal", func() { + sc := client.ServiceClient(fakeServer) + err := AbortMigration(ctx, sc, "inst-uuid-1", 42) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when Nova returns a server error", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("DELETE /servers/inst-uuid-1/migrations/42", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"error": "Internal Server Error"}`) + }) + }) + + It("should propagate the error", func() { + sc := client.ServiceClient(fakeServer) + err := AbortMigration(ctx, sc, "inst-uuid-1", 42) + Expect(err).To(HaveOccurred()) + }) + }) + }) + + Describe("SettleIncomingMigrations", func() { + Context("when there are no incoming migrations", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(emptyMigrations)) + }) + + It("should return empty aborted and waiting", func() { + sc := client.ServiceClient(fakeServer) + aborted, waiting, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(aborted).To(BeEmpty()) + Expect(waiting).To(BeEmpty()) + }) + }) + + Context("when there are abortable migrations", func() { + var deleteCalls int + + BeforeEach(func() { + deleteCalls = 0 + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 1, + "uuid": "mig-1", + "instance_uuid": "inst-1", + "status": "running", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 2, + "uuid": "mig-2", + "instance_uuid": "inst-2", + "status": "queued", + "source_compute": "node007-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + + fakeServer.Mux.HandleFunc("DELETE /servers/inst-1/migrations/1", func(w http.ResponseWriter, r *http.Request) { + deleteCalls++ + w.WriteHeader(http.StatusAccepted) + }) + + fakeServer.Mux.HandleFunc("DELETE /servers/inst-2/migrations/2", func(w http.ResponseWriter, r *http.Request) { + deleteCalls++ + w.WriteHeader(http.StatusAccepted) + }) + }) + + It("should abort all and return them in aborted list", func() { + sc := client.ServiceClient(fakeServer) + aborted, waiting, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(aborted).To(HaveLen(2)) + Expect(waiting).To(BeEmpty()) + Expect(deleteCalls).To(Equal(2)) + }) + }) + + Context("when there is a post-migrating migration (not abortable)", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 99, + "uuid": "mig-99", + "instance_uuid": "inst-99", + "status": "post-migrating", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + }) + + It("should return in waiting list without issuing DELETE", func() { + sc := client.ServiceClient(fakeServer) + aborted, waiting, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(aborted).To(BeEmpty()) + Expect(waiting).To(HaveLen(1)) + Expect(waiting[0].ID).To(Equal(99)) + }) + }) + + Context("with a mixed set (abortable + post-migrating + outbound + completed)", func() { + var deleteCalls int + + BeforeEach(func() { + deleteCalls = 0 + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 1, + "uuid": "mig-1", + "instance_uuid": "inst-1", + "status": "running", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 2, + "uuid": "mig-2", + "instance_uuid": "inst-2", + "status": "preparing", + "source_compute": "node007-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 3, + "uuid": "mig-3", + "instance_uuid": "inst-3", + "status": "post-migrating", + "source_compute": "node002-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + }, + { + "id": 4, + "uuid": "mig-4", + "instance_uuid": "inst-4", + "status": "running", + "source_compute": "node009-bb549", + "dest_compute": "node005-bb549", + "migration_type": "live-migration" + }, + { + "id": 5, + "uuid": "mig-5", + "instance_uuid": "inst-5", + "status": "completed", + "source_compute": "node003-bb549", + "dest_compute": "node009-bb549", + "migration_type": "live-migration" + } + ]}`)) + + fakeServer.Mux.HandleFunc("DELETE /servers/inst-1/migrations/1", func(w http.ResponseWriter, r *http.Request) { + deleteCalls++ + w.WriteHeader(http.StatusAccepted) + }) + + fakeServer.Mux.HandleFunc("DELETE /servers/inst-2/migrations/2", func(w http.ResponseWriter, r *http.Request) { + deleteCalls++ + w.WriteHeader(http.StatusAccepted) + }) + }) + + It("should abort 2, wait on 1, ignore outbound and completed", func() { + sc := client.ServiceClient(fakeServer) + aborted, waiting, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(aborted).To(HaveLen(2)) + Expect(waiting).To(HaveLen(1)) + Expect(waiting[0].ID).To(Equal(3)) + Expect(deleteCalls).To(Equal(2)) + }) + }) + + Context("when Nova list returns an error", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, `{"error": "Internal Server Error"}`) + }) + }) + + It("should propagate the error", func() { + sc := client.ServiceClient(fakeServer) + _, _, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when there is a running evacuation (not abortable via migration-abort)", func() { + BeforeEach(func() { + fakeServer.Mux.HandleFunc("GET /os-migrations", migrationsHandler(`{"migrations": [ + { + "id": 88, + "uuid": "mig-evac-1", + "instance_uuid": "inst-evac-1", + "status": "running", + "source_compute": "node001-bb549", + "dest_compute": "node009-bb549", + "migration_type": "evacuation" + } + ]}`)) + }) + + It("should place the evacuation in waiting without issuing DELETE", func() { + sc := client.ServiceClient(fakeServer) + aborted, waiting, err := SettleIncomingMigrations(ctx, sc, "node009-bb549") + Expect(err).NotTo(HaveOccurred()) + Expect(aborted).To(BeEmpty()) + Expect(waiting).To(HaveLen(1)) + Expect(waiting[0].ID).To(Equal(88)) + Expect(waiting[0].MigrationType).To(Equal("evacuation")) + }) + }) + }) +})