diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index ff2e66b93..5fa3230e0 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -159,6 +159,16 @@ spec: matching the request's project, resource group, and availability zone, with enough free memory capacity for the requested VM. Hosts without a matching reservation or without enough free capacity receive a lower weight. + - name: kvm_cr_migration_slot + description: | + During live migrations of VMs that occupy a committed-resource reservation + slot, this weigher boosts hosts that either have a ready matching CR slot + with enough remaining capacity for the full slot size, or have enough free + memory for the slot to follow via the reconciler. Hosts that can accommodate + neither receive a lower weight. If the VM has no confirmed CR slot, all + candidates receive equal weight. + Only activates for live_migrate requests. All other intents pass through. + Emits cortex_nova_weigh_cr_migration_slot_requests_total. --- apiVersion: cortex.cloud/v1alpha1 kind: Pipeline @@ -316,6 +326,16 @@ spec: matching the request's project, resource group, and availability zone, with enough free memory capacity for the requested VM. Hosts without a matching reservation or without enough free capacity receive a lower weight. + - name: kvm_cr_migration_slot + description: | + During live migrations of VMs that occupy a committed-resource reservation + slot, this weigher boosts hosts that either have a ready matching CR slot + with enough remaining capacity for the full slot size, or have enough free + memory for the slot to follow via the reconciler. Hosts that can accommodate + neither receive a lower weight. If the VM has no confirmed CR slot, all + candidates receive equal weight. + Only activates for live_migrate requests. All other intents pass through. + Emits cortex_nova_weigh_cr_migration_slot_requests_total. --- apiVersion: cortex.cloud/v1alpha1 kind: Pipeline @@ -786,6 +806,16 @@ spec: matching the request's project, resource group, and availability zone, with enough free memory capacity for the requested VM. Hosts without a matching reservation or without enough free capacity receive a lower weight. + - name: kvm_cr_migration_slot + description: | + During live migrations of VMs that occupy a committed-resource reservation + slot, this weigher boosts hosts that either have a ready matching CR slot + with enough remaining capacity for the full slot size, or have enough free + memory for the slot to follow via the reconciler. Hosts that can accommodate + neither receive a lower weight. If the VM has no confirmed CR slot, all + candidates receive equal weight. + Only activates for live_migrate requests. All other intents pass through. + Emits cortex_nova_weigh_cr_migration_slot_requests_total. --- apiVersion: cortex.cloud/v1alpha1 kind: Pipeline @@ -943,4 +973,14 @@ spec: matching the request's project, resource group, and availability zone, with enough free memory capacity for the requested VM. Hosts without a matching reservation or without enough free capacity receive a lower weight. + - name: kvm_cr_migration_slot + description: | + During live migrations of VMs that occupy a committed-resource reservation + slot, this weigher boosts hosts that either have a ready matching CR slot + with enough remaining capacity for the full slot size, or have enough free + memory for the slot to follow via the reconciler. Hosts that can accommodate + neither receive a lower weight. If the VM has no confirmed CR slot, all + candidates receive equal weight. + Only activates for live_migrate requests. All other intents pass through. + Emits cortex_nova_weigh_cr_migration_slot_requests_total. {{- end }} diff --git a/internal/scheduling/nova/crs/evaluator.go b/internal/scheduling/nova/crs/evaluator.go index bfd5f9607..67faef722 100644 --- a/internal/scheduling/nova/crs/evaluator.go +++ b/internal/scheduling/nova/crs/evaluator.go @@ -24,6 +24,19 @@ type SlotEvaluator struct { // BuildSlotEvaluator lists HV CRDs and CR Reservation CRDs once and returns an evaluator // that can answer slot-usability queries without further K8s reads. func BuildSlotEvaluator(ctx context.Context, c client.Client) (*SlotEvaluator, error) { + var resList v1alpha1.ReservationList + if err := c.List(ctx, &resList, + client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource}, + ); err != nil { + return nil, err + } + return BuildSlotEvaluatorFromReservations(ctx, c, resList.Items) +} + +// BuildSlotEvaluatorFromReservations builds a SlotEvaluator from an already-fetched +// reservation slice. Use this when the caller has already listed reservations to avoid +// a redundant K8s read. +func BuildSlotEvaluatorFromReservations(ctx context.Context, c client.Client, reservations []v1alpha1.Reservation) (*SlotEvaluator, error) { eval := &SlotEvaluator{ hvFreeMemory: make(map[string]int64), reservationsByHost: make(map[string][]v1alpha1.Reservation), @@ -45,13 +58,7 @@ func BuildSlotEvaluator(ctx context.Context, c client.Client) (*SlotEvaluator, e eval.hvFreeMemory[hv.Name] = max(effectiveMemQ.Value()-allocMemQ.Value(), 0) } - var resList v1alpha1.ReservationList - if err := c.List(ctx, &resList, - client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource}, - ); err != nil { - return nil, err - } - for _, res := range resList.Items { + for _, res := range reservations { if !res.IsReady() { continue } @@ -104,6 +111,35 @@ func (e *SlotEvaluator) HasUsableSlot(hostName, projectID, flavorGroup string, v return false } +// HasSlotWithCapacity reports whether hostName has at least one ready CR slot +// matching projectID + flavorGroup whose remaining memory is >= requiredBytes. +// Unlike HasUsableSlot, this does not apply the overfill model — it is used +// during migration slot filtering where the full slot size must fit within a +// single reservation on the target host. +func (e *SlotEvaluator) HasSlotWithCapacity(hostName, projectID, flavorGroup string, requiredBytes int64) bool { + for _, slot := range e.SlotsForHost(hostName, projectID, flavorGroup) { + if ReservationRemainingMemory(slot) >= requiredBytes { + return true + } + } + return false +} + +// CanAccommodateSlot reports whether hostName has enough free memory to absorb +// a reservation block of requiredBytes. Used to check whether a slot can follow +// a migrating VM to this host via the reconciler, even when no existing +// compatible slot is present. +// +// Free memory is computed as: hvFreeMemory - sum(all ready committed-resource reservation blocks on host). +func (e *SlotEvaluator) CanAccommodateSlot(hostName string, requiredBytes int64) bool { + var allBlocks int64 + for _, res := range e.reservationsByHost[hostName] { + blockQ := res.Spec.Resources[hv1.ResourceMemory] + allBlocks += blockQ.Value() + } + return e.hvFreeMemory[hostName]-allBlocks >= requiredBytes +} + // ReservationRemainingMemory returns how many bytes of memory remain // unallocated in a reservation slot. Returns 0 if the slot is full or nil. func ReservationRemainingMemory(res v1alpha1.Reservation) int64 { diff --git a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go new file mode 100644 index 000000000..56f09651f --- /dev/null +++ b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot.go @@ -0,0 +1,160 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package weighers + +import ( + "context" + "log/slog" + + "sigs.k8s.io/controller-runtime/pkg/client" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" +) + +// Options for the KVM CR migration slot weigher. +type KVMCRMigrationSlotOpts struct { + // Weight assigned to hosts that can accommodate the CR reservation slot. + // Default: 0.1 + SlotHostWeight *float64 `json:"slotHostWeight,omitempty"` + // Weight assigned to hosts that cannot accommodate the CR reservation slot. + // Default: 0.0 + DefaultHostWeight *float64 `json:"defaultHostWeight,omitempty"` +} + +func (o KVMCRMigrationSlotOpts) Validate() error { + return nil +} + +func (o KVMCRMigrationSlotOpts) GetSlotHostWeight() float64 { + if o.SlotHostWeight == nil { + return 0.1 + } + return *o.SlotHostWeight +} + +func (o KVMCRMigrationSlotOpts) GetDefaultHostWeight() float64 { + if o.DefaultHostWeight == nil { + return 0.0 + } + return *o.DefaultHostWeight +} + +// KVMCRMigrationSlotStep weighs live-migration candidates by whether they can +// accommodate the CR reservation slot of the migrating VM. +// +// When a VM with a CR reservation slot is migrated, this weigher boosts hosts +// that either have a ready CR reservation with sufficient remaining capacity for +// the slot, or have enough free memory for the slot to follow via the reconciler. +// This steers the migration toward hosts where double-blocking is avoided or +// minimised. +// +// If the VM has no confirmed CR reservation, all candidates receive equal weight +// (no effect on ranking). +// +// Only activates for LiveMigrationIntent. +type KVMCRMigrationSlotStep struct { + lib.BaseWeigher[api.ExternalSchedulerRequest, KVMCRMigrationSlotOpts] +} + +func (s *KVMCRMigrationSlotStep) Run( + traceLog *slog.Logger, + request api.ExternalSchedulerRequest, +) (*lib.FilterWeigherPipelineStepResult, error) { + + result := s.IncludeAllHostsFromRequest(request) + + intent, err := request.GetIntent() + if err != nil || intent != api.LiveMigrationIntent { + traceLog.Info("not a live migration, skipping CR migration slot weigher") + return result, nil //nolint:nilerr + } + + instanceUUID := request.Spec.Data.InstanceUUID + projectID := request.Spec.Data.ProjectID + + var allReservations v1alpha1.ReservationList + if err := s.Client.List(context.Background(), &allReservations, + client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource}, + ); err != nil { + return nil, err + } + + // Find the source slot that has this VM confirmed. + var sourceSlot *v1alpha1.Reservation + for i := range allReservations.Items { + res := &allReservations.Items[i] + if !res.IsReady() { + continue + } + if res.Status.CommittedResourceReservation == nil || res.Spec.CommittedResourceReservation == nil { + continue + } + if _, ok := res.Status.CommittedResourceReservation.Allocations[instanceUUID]; ok { + sourceSlot = res + break + } + } + + if sourceSlot == nil { + traceLog.Info("migrating VM has no confirmed CR reservation slot, skipping slot weigher", + "instanceUUID", instanceUUID) + return result, nil + } + + slotMemoryBytes := sourceSlot.Spec.Resources[hv1.ResourceMemory] + if slotMemoryBytes.IsZero() { + traceLog.Info("source CR slot has no memory resource, skipping slot weigher", + "instanceUUID", instanceUUID, + "reservation", sourceSlot.Name) + return result, nil + } + + resourceGroup := sourceSlot.Spec.CommittedResourceReservation.ResourceGroup + + traceLog.Info("found source CR reservation slot for migrating VM", + "instanceUUID", instanceUUID, + "reservation", sourceSlot.Name, + "slotMemoryBytes", slotMemoryBytes.Value(), + "resourceGroup", resourceGroup, + ) + + evaluator, err := crs.BuildSlotEvaluatorFromReservations(context.Background(), s.Client, allReservations.Items) + if err != nil { + return nil, err + } + + slotHostWeight := s.Options.GetSlotHostWeight() + defaultHostWeight := s.Options.GetDefaultHostWeight() + + for host := range result.Activations { + hasSlot := evaluator.HasSlotWithCapacity(host, projectID, resourceGroup, slotMemoryBytes.Value()) + canFit := evaluator.CanAccommodateSlot(host, slotMemoryBytes.Value()) + switch { + case hasSlot: + result.Activations[host] = slotHostWeight + traceLog.Info("host has existing CR slot for migration, boosting weight", + "host", host, "weight", slotHostWeight) + case canFit: + result.Activations[host] = slotHostWeight + traceLog.Info("host can accommodate slot via reconciler, boosting weight", + "host", host, "weight", slotHostWeight) + default: + result.Activations[host] = defaultHostWeight + traceLog.Info("host cannot accommodate CR slot, applying low weight", + "host", host, "weight", defaultHostWeight) + } + } + + return result, nil +} + +func init() { + Index["kvm_cr_migration_slot"] = func() NovaWeigher { + return &KVMCRMigrationSlotStep{} + } +} diff --git a/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go new file mode 100644 index 000000000..4ff8d3f21 --- /dev/null +++ b/internal/scheduling/nova/plugins/weighers/kvm_cr_migration_slot_test.go @@ -0,0 +1,254 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package weighers + +import ( + "log/slog" + "testing" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const ( + testInstanceUUID = "vm-migrating" + testProjectID = "proj-1" + testSourceHost = "host-src" + testResourceGroup = "hana-v2" +) + +// newCRMigrationSlotWeigher builds a KVMCRMigrationSlotStep backed by a fake client. +func newCRMigrationSlotWeigher(t *testing.T, opts KVMCRMigrationSlotOpts, objs ...client.Object) *KVMCRMigrationSlotStep { + t.Helper() + scheme := buildTestScheme(t) + step := &KVMCRMigrationSlotStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() + step.Options = opts + return step +} + +// migrationRequest builds a live-migration request from the given candidate hosts. +func migrationRequest(hosts ...string) api.ExternalSchedulerRequest { + hostList := make([]api.ExternalSchedulerHost, len(hosts)) + for i, h := range hosts { + hostList[i] = api.ExternalSchedulerHost{ComputeHost: h} + } + return api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + InstanceUUID: testInstanceUUID, + ProjectID: testProjectID, + SchedulerHints: map[string]any{ + "_nova_check_type": "live_migrate", + }, + }, + }, + Hosts: hostList, + } +} + +// confirmedSourceSlot builds a ready CR reservation with testInstanceUUID confirmed in Status. +func confirmedSourceSlot(memory string) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "slot-src-" + testSourceHost, + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: testSourceHost, + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse(memory), + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + ProjectID: testProjectID, + ResourceGroup: testResourceGroup, + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: testSourceHost, + Conditions: []metav1.Condition{ + {Type: v1alpha1.ReservationConditionReady, Status: metav1.ConditionTrue, Reason: "ReservationActive"}, + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationStatus{ + Allocations: map[string]string{testInstanceUUID: testSourceHost}, + }, + }, + } +} + +// emptyTargetSlot builds a ready CR reservation with no VM allocations on the given host. +func emptyTargetSlot(name, host, resourceGroup, memory string) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: host, + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse(memory), + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + ProjectID: "proj-1", + ResourceGroup: resourceGroup, + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: host, + Conditions: []metav1.Condition{ + {Type: v1alpha1.ReservationConditionReady, Status: metav1.ConditionTrue, Reason: "ReservationActive"}, + }, + }, + } +} + +// hvWithFreeMemory builds a Hypervisor with the given effective capacity and zero allocation. +func hvWithFreeMemory(name, memory string) *hv1.Hypervisor { + return &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse(memory), + }, + Allocation: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse("0"), + }, + }, + } +} + +func TestKVMCRMigrationSlotStep_Run(t *testing.T) { + defaultOpts := KVMCRMigrationSlotOpts{SlotHostWeight: floatPtr(1.0), DefaultHostWeight: floatPtr(0.1)} + + tests := []struct { + name string + objects []client.Object + request api.ExternalSchedulerRequest + opts KVMCRMigrationSlotOpts + expectedWeights map[string]float64 + }{ + { + name: "non-migration intent: all hosts get no-effect weight", + objects: []client.Object{ + confirmedSourceSlot("16Gi"), + }, + request: api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + InstanceUUID: testInstanceUUID, + ProjectID: testProjectID, + // no _nova_check_type → CreateIntent + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host-a"}, {ComputeHost: "host-b"}}, + }, + opts: defaultOpts, + expectedWeights: map[string]float64{"host-a": 0.0, "host-b": 0.0}, + }, + { + name: "no source slot for VM: all candidates get no-effect weight", + objects: []client.Object{}, + request: migrationRequest("host-a", "host-b"), + opts: defaultOpts, + expectedWeights: map[string]float64{"host-a": 0.0, "host-b": 0.0}, + }, + { + name: "host with matching slot gets slot weight, others get default weight", + objects: []client.Object{ + confirmedSourceSlot("16Gi"), + emptyTargetSlot("slot-a", "host-a", "hana-v2", "16Gi"), + emptyTargetSlot("slot-b", "host-b", "hana-v2", "8Gi"), // too small + }, + request: migrationRequest("host-a", "host-b", "host-c"), + opts: defaultOpts, + expectedWeights: map[string]float64{"host-a": 1.0, "host-b": 0.1, "host-c": 0.1}, + }, + { + name: "no compatible slot on any candidate: hosts penalised (no capacity)", + objects: []client.Object{ + confirmedSourceSlot("16Gi"), + }, + request: migrationRequest("host-a", "host-b"), + opts: defaultOpts, + expectedWeights: map[string]float64{"host-a": 0.1, "host-b": 0.1}, + }, + { + name: "host with free capacity but no slot: boosted via accommodate path", + objects: []client.Object{ + confirmedSourceSlot("16Gi"), + hvWithFreeMemory("host-a", "32Gi"), // enough free memory for the slot + hvWithFreeMemory("host-b", "8Gi"), // too small for the slot + }, + request: migrationRequest("host-a", "host-b"), + opts: defaultOpts, + expectedWeights: map[string]float64{"host-a": 1.0, "host-b": 0.1}, + }, + { + name: "wrong resource group on target: no slot match, falls back to capacity check", + objects: []client.Object{ + confirmedSourceSlot("16Gi"), + emptyTargetSlot("slot-a", "host-a", "general-v3", "16Gi"), + }, + request: migrationRequest("host-a"), + opts: defaultOpts, + expectedWeights: map[string]float64{"host-a": 0.1}, + }, + { + name: "source slot has zero memory: no-effect weight for all", + objects: []client.Object{ + // source slot with no memory resource + func() *v1alpha1.Reservation { + res := confirmedSourceSlot("0") + res.Spec.Resources = map[hv1.ResourceName]resource.Quantity{} // no memory key + return res + }(), + emptyTargetSlot("slot-a", "host-a", "hana-v2", "16Gi"), + }, + request: migrationRequest("host-a"), + opts: defaultOpts, + expectedWeights: map[string]float64{"host-a": 0.0}, + }, + { + name: "nil opts use default weights", + objects: []client.Object{ + confirmedSourceSlot("16Gi"), + emptyTargetSlot("slot-a", "host-a", "hana-v2", "16Gi"), + }, + request: migrationRequest("host-a", "host-b"), + opts: KVMCRMigrationSlotOpts{}, // nil → defaults: slot=0.1, default=0.0 + expectedWeights: map[string]float64{"host-a": 0.1, "host-b": 0.0}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + weigher := newCRMigrationSlotWeigher(t, tt.opts, tt.objects...) + result, err := weigher.Run(slog.Default(), tt.request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for host, expected := range tt.expectedWeights { + actual := result.Activations[host] + if actual != expected { + t.Errorf("host %q: expected weight %v, got %v", host, expected, actual) + } + } + if len(result.Activations) != len(tt.expectedWeights) { + t.Errorf("expected %d hosts in activations, got %d: %v", + len(tt.expectedWeights), len(result.Activations), result.Activations) + } + }) + } +}