From e5230e188af6a08d91351f5d7168119696282431 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Thu, 15 Oct 2026 16:45:36 +0200 Subject: [PATCH 01/20] feat(nova): improve virtual machine creation logging Log the specific reason (with instance UUID) whenever VM creation is skipped, and log successful create/update events in the vm handler so resource processing can be traced. Signed-off-by: Philipp Matthes --- .../filter_weigher_pipeline_controller.go | 26 ++++++++++++------- internal/scheduling/nova/vm_handler.go | 2 ++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller.go b/internal/scheduling/nova/filter_weigher_pipeline_controller.go index 5eee060e4..f63eed57b 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller.go @@ -214,21 +214,29 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision c.upsertHistory(ctx, decision, err) } - // We create a virtual machine resource based on the scheduling decision if ... - createVM := err == nil && // The pipeline was executed successfully - !request.Options.ReadOnly && // The request is not read-only - c.FeatureGates.VirtualMachineCreation && // VM creation is enabled - slices.Contains(api.NovaIntents, decision.Spec.Intent) && // Triggered by Nova - request.IsKVMRequest() // Supported hypervisor type - if createVM { + // We create a virtual machine resource based on the scheduling decision, + // logging the specific reason whenever we skip creation so it can be traced. + instanceUUID := request.Spec.Data.InstanceUUID + switch { + case err != nil: + log.V(1).Info("skipping virtual machine creation, pipeline run failed", "instanceUUID", instanceUUID) + case request.Options.ReadOnly: + log.V(1).Info("skipping virtual machine creation, request is read-only", "instanceUUID", instanceUUID) + case !c.FeatureGates.VirtualMachineCreation: + log.V(1).Info("skipping virtual machine creation, feature gate disabled", "instanceUUID", instanceUUID) + case !slices.Contains(api.NovaIntents, decision.Spec.Intent): + log.V(1).Info("skipping virtual machine creation, intent not triggered by Nova", "instanceUUID", instanceUUID, "intent", decision.Spec.Intent) + case !request.IsKVMRequest(): + log.V(1).Info("skipping virtual machine creation, unsupported hypervisor type", "instanceUUID", instanceUUID) + default: vmUpsertErr := c.vmHandler.Upsert(ctx, decision, request) if c.FeatureGates.VirtualMachineCreationMandatory && vmUpsertErr != nil { - log.Error(vmUpsertErr, "failed to upsert virtual machine resource and it is mandatory") + log.Error(vmUpsertErr, "failed to upsert virtual machine resource and it is mandatory", "instanceUUID", instanceUUID) return &request, vmUpsertErr } if vmUpsertErr != nil { // Non-fatal: the placement decision itself succeeded. - log.Error(vmUpsertErr, "failed to upsert virtual machine resource") + log.Error(vmUpsertErr, "failed to upsert virtual machine resource", "instanceUUID", instanceUUID) } } diff --git a/internal/scheduling/nova/vm_handler.go b/internal/scheduling/nova/vm_handler.go index 970bd88e0..4b48fedd2 100644 --- a/internal/scheduling/nova/vm_handler.go +++ b/internal/scheduling/nova/vm_handler.go @@ -188,6 +188,7 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.D if err := h.Status().Update(ctx, vm); err != nil { return fmt.Errorf("failed to update VM status: %w", err) } + log.Info("created virtual machine resource", "instanceUUID", vm.Name, "targetHost", targetHost, "intent", intent) return nil } else if err != nil { return fmt.Errorf("error talking to apiserver: %w", err) @@ -229,5 +230,6 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.D if err := h.Status().Update(ctx, vmInCluster); err != nil { return fmt.Errorf("failed to update VM status: %w", err) } + log.Info("updated virtual machine resource", "instanceUUID", vm.Name, "targetHost", targetHost, "intent", intent) return nil } From 8ea290be7e5b53f89285bfb1a38edc81147d3e1d Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Thu, 15 Oct 2026 17:29:22 +0200 Subject: [PATCH 02/20] Fix broken feature gate configuration in nova bundle Signed-off-by: Philipp Matthes --- helm/bundles/cortex-nova/values.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 312c73b38..6db8c0322 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -193,15 +193,14 @@ cortex-scheduling-controllers: - nova-history-cleanup-task - commitments-sync-task # required for committed resources # Feature gates control optional capabilities within running components. - featureGates: - # Enables VM allocation write-back into Reservation slots and no-host-found - # classification by committed resource coverage. Enable only on deployments - # that use committed resources. Requires also enabling of CR controllers and tasks - committedResourceTracking: false - # Enables the creation of VM resources based on the scheduling spec for - # non-read-only requests. - virtualMachineCreation: true - virtualMachineCreationMandatory: false + # Enables VM allocation write-back into Reservation slots and no-host-found + # classification by committed resource coverage. Enable only on deployments + # that use committed resources. Requires also enabling of CR controllers and tasks + committedResourceTracking: false + # Enables the creation of VM resources based on the scheduling spec for + # non-read-only requests. + virtualMachineCreation: true + virtualMachineCreationMandatory: false # Pipeline used for the empty-state capacity probe (ignores allocations and reservations). capacityTotalPipeline: "kvm-general-purpose-load-balancing" # Pipeline used for the current-state capacity probe (considers current VM allocations). From 900e6ef23cc298c675887a87288988f2b398d357 Mon Sep 17 00:00:00 2001 From: Marcel <156897072+mblos@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:23:45 +0200 Subject: [PATCH 03/20] fix: CRs confirm superseded/expired CRs correctly within CR transactions (#1372) Fixes `CortexNovaCommittedResourceTimeoutsTooHigh` alerts when a batch carries a superseded/expired commitment. Also adds a hang guard for cores commitments --------- Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com> --- .../commitments/api/change_commitments.go | 35 +++++++- .../api/change_commitments_test.go | 39 +++++++++ .../committed_resource_controller.go | 81 +++++++++++++++++-- .../commitments/reservation_manager.go | 6 +- 4 files changed, 151 insertions(+), 10 deletions(-) diff --git a/internal/scheduling/reservations/commitments/api/change_commitments.go b/internal/scheduling/reservations/commitments/api/change_commitments.go index 0aec12167..1a1b601d2 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments.go @@ -74,6 +74,25 @@ func transferredCommitmentUUIDs(req liquid.CommitmentChangeRequest) map[liquid.C return transfers } +// affectedCRNames returns the sorted CommittedResource CRD names (commitment-) touched +// by this request, so the entry log pinpoints which CRs to grep for during debugging. +func affectedCRNames(req liquid.CommitmentChangeRequest) []string { + seen := make(map[liquid.CommitmentUUID]struct{}) + for _, projectChanges := range req.ByProject { + for _, resourceChanges := range projectChanges.ByResource { + for _, c := range resourceChanges.Commitments { + seen[c.UUID] = struct{}{} + } + } + } + names := make([]string, 0, len(seen)) + for uuid := range seen { + names = append(names, "commitment-"+string(uuid)) + } + sort.Strings(names) + return names +} + // crWatch pairs a CRD name with the generation written by the API so the polling loop // can skip cache reads that have not yet reflected the write (stale-cache guard). type crWatch struct { @@ -139,7 +158,11 @@ func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Reque return } - logger.Info("received change commitments request", "affectedProjects", len(req.ByProject), "dryRun", req.DryRun, "availabilityZone", req.AZ) + logger.Info("received change commitments request", + "affectedProjects", len(req.ByProject), + "affectedCRs", affectedCRNames(req), + "dryRun", req.DryRun, + "availabilityZone", req.AZ) if req.AZ == "" { statusCode = http.StatusBadRequest @@ -377,7 +400,15 @@ ProcessLoop: break ProcessLoop } - toWatch = append(toWatch, crWatch{name: crName, generation: crGeneration}) + // Superseded/expired transitions reconcile to Ready=False with reason + // "superseded"/"expired", which is not a terminal outcome the poller + // recognizes — polling them runs out the full WatchTimeout and rolls back + // the confirmations batched alongside them. Limes needs no Ready outcome for + // an inactive transition, so snapshot it for rollback but do not poll it. + if stateDesired.State != v1alpha1.CommitmentStatusSuperseded && + stateDesired.State != v1alpha1.CommitmentStatusExpired { + toWatch = append(toWatch, crWatch{name: crName, generation: crGeneration}) + } snapshots = append(snapshots, snap) logger.V(1).Info("upserted CommittedResource CRD", "name", crName) } diff --git a/internal/scheduling/reservations/commitments/api/change_commitments_test.go b/internal/scheduling/reservations/commitments/api/change_commitments_test.go index acd10bb80..a387c60f7 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments_test.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments_test.go @@ -442,6 +442,45 @@ func TestHandleChangeCommitments(t *testing.T) { ExpectedAllowRejection: map[string]bool{"commitment-uuid-to-sup": true}, ExpectedCRSpecs: map[string]int64{"commitment-uuid-to-sup": 0}, }, + // --- Regression: superseded/expired transitions must not be polled --- + { + // Batch combining a confirmation and a confirmed→superseded transition. The superseded CR + // gets no poller-terminal condition (NoCondition), mirroring the real controller which + // stamps Ready=False/Reason="superseded" — a reason the poller does not accept. Before the + // fix the superseded CR was added to toWatch, ran out the full WatchTimeout, and rolled back + // the good confirmation batched alongside it (HTTP 500, firing CortexNovaCommittedResource- + // TimeoutsTooHigh). With the fix it is snapshotted for rollback but not polled, so the + // confirmation is accepted and nothing is rolled back. + Name: "Batch confirm + superseded with no terminal condition: superseded not polled, confirmation accepted, no rollback", + Flavors: []*TestFlavor{m1Small}, + ExistingCRs: []*TestCR{ + {CommitmentUUID: "uuid-sup-batch", State: v1alpha1.CommitmentStatusConfirmed, AmountMiB: 1024, ProjectID: "project-A", AZ: "az-a"}, + }, + NoCondition: []string{"commitment-uuid-sup-batch"}, + CommitmentRequest: newCommitmentRequest("az-a", false, 1234, + createCommitment("hw_version_hana_1_ram", "project-B", "uuid-conf-batch", "confirmed", 2), + TestCommitment{ + ResourceName: "hw_version_hana_1_ram", + ProjectID: "project-A", + ConfirmationID: "uuid-sup-batch", + OldState: "confirmed", + State: "superseded", + Amount: 1, + }), + // Short timeout: if the superseded CR were still polled, the request would run out the + // WatchTimeout and roll back, failing this test fast instead of hanging for the default. + CustomConfig: func() *commitments.APIConfig { + cfg := commitments.DefaultAPIConfig() + cfg.WatchTimeout = metav1.Duration{Duration: 2 * time.Second} + cfg.WatchPollInterval = metav1.Duration{Duration: 50 * time.Millisecond} + cfg.FlavorGroupResourceConfig = map[string]commitments.FlavorGroupResourcesConfig{ + "*": {RAM: commitments.RAMResourceTypeConfig{HandlesCommitments: true, HasCapacity: true}}, + } + return &cfg + }(), + ExpectedAPIResponse: newAPIResponse(), + ExpectedCreatedCRNames: []string{"commitment-uuid-conf-batch", "commitment-uuid-sup-batch"}, + }, // --- Resize --- { Name: "Resize down: confirmed→confirmed with less capacity, RequiresConfirmation=true", diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go index f8703d734..b3a438090 100644 --- a/internal/scheduling/reservations/commitments/committed_resource_controller.go +++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go @@ -170,7 +170,23 @@ func (r *CommittedResourceController) reconcilePending(ctx context.Context, logg return ctrl.Result{RequeueAfter: 3 * time.Second}, nil } logger.Info("committed resource accepted", "generation", cr.Generation, "amount", cr.Spec.Amount.String()) - return ctrl.Result{}, r.setAccepted(ctx, cr) + initialGen := cr.Generation + if err := r.setAccepted(ctx, cr); err != nil { + return ctrl.Result{}, err + } + var latest v1alpha1.CommittedResource + if err := r.Get(ctx, client.ObjectKeyFromObject(cr), &latest); err != nil { + logger.V(1).Info("re-Get after accept failed, requeueing", "err", err) + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + if latest.Generation > initialGen { + logger.Info("generation advanced during reconcile, requeueing", + "processedGen", initialGen, + "latestGen", latest.Generation, + ) + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + return ctrl.Result{}, nil } func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (ctrl.Result, error) { @@ -199,7 +215,23 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo allReady, anyFailed, _, _, _, _, checkErr := r.checkChildReservationStatus(ctx, cr, result.TotalSlots, false) if checkErr == nil && allReady && !anyFailed { logger.Info("recovered from spurious rejection: all slots ready", "generation", cr.Generation) - return ctrl.Result{}, r.setAccepted(ctx, cr) + initialGen := cr.Generation + if err := r.setAccepted(ctx, cr); err != nil { + return ctrl.Result{}, err + } + var latest v1alpha1.CommittedResource + if err := r.Get(ctx, client.ObjectKeyFromObject(cr), &latest); err != nil { + logger.V(1).Info("re-Get after accept failed, requeueing", "err", err) + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + if latest.Generation > initialGen { + logger.Info("generation advanced during reconcile, requeueing", + "processedGen", initialGen, + "latestGen", latest.Generation, + ) + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + return ctrl.Result{}, nil } if checkErr != nil { logger.V(1).Info("spurious-rejection recovery: slot check failed, will retry", "err", checkErr) @@ -273,13 +305,21 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo return ctrl.Result{RequeueAfter: 3 * time.Second}, nil } logger.Info("committed resource accepted", "generation", cr.Generation, "amount", cr.Spec.Amount.String()) + // setAccepted mutates cr in-place via the Patch response (server returns the latest object, + // which may already carry a newer generation from a concurrent spec write). + // Capture initialGen before the mutation so the comparison below is correct. + initialGen := cr.Generation if err := r.setAccepted(ctx, cr); err != nil { return ctrl.Result{}, err } var latest v1alpha1.CommittedResource - if err := r.Get(ctx, client.ObjectKeyFromObject(cr), &latest); err == nil && latest.Generation > cr.Generation { + if err := r.Get(ctx, client.ObjectKeyFromObject(cr), &latest); err != nil { + logger.V(1).Info("re-Get after accept failed, requeueing", "err", err) + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + if latest.Generation > initialGen { logger.Info("generation advanced during reconcile, requeueing", - "processedGen", cr.Generation, + "processedGen", initialGen, "latestGen", latest.Generation, ) return ctrl.Result{RequeueAfter: 1 * time.Second}, nil @@ -382,7 +422,23 @@ func (r *CommittedResourceController) reconcileCoresHeadroom(ctx context.Context } logger.Info("cores commitment accepted", "generation", cr.Generation, "cores", requestedCores) - return ctrl.Result{}, r.setAccepted(ctx, cr) + initialGen := cr.Generation + if err := r.setAccepted(ctx, cr); err != nil { + return ctrl.Result{}, err + } + var latest v1alpha1.CommittedResource + if err := r.Get(ctx, client.ObjectKeyFromObject(cr), &latest); err != nil { + logger.V(1).Info("re-Get after accept failed, requeueing", "err", err) + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + if latest.Generation > initialGen { + logger.Info("generation advanced during reconcile, requeueing", + "processedGen", initialGen, + "latestGen", latest.Generation, + ) + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + return ctrl.Result{}, nil } func (r *CommittedResourceController) applyReservationState(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (*ApplyResult, error) { @@ -683,6 +739,21 @@ func (r *CommittedResourceController) setNotReadyRetry(ctx context.Context, cr * // use this on the "apply succeeded, waiting for slots" path so the watch suppression gate // does not fire while placement is working. func (r *CommittedResourceController) patchNotReady(ctx context.Context, cr *v1alpha1.CommittedResource, reason, message string, resetTimer bool) error { + // Skip the write when nothing would change. Without this guard, every reconcile + // in a steady Reserving/etc. state issues a patch because computeStatusSummary + // receives time.Now() and may produce a different string, triggering the + // CommittedResource self-watch and causing a continuous reconcile loop. + // resetTimer=true forces a timestamp refresh — skip the guard in that case. + if !resetTimer { + cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond != nil && + cond.Status == metav1.ConditionFalse && + cond.Reason == reason && + cond.Message == message && + cond.ObservedGeneration == cr.Generation { + return nil + } + } old := cr.DeepCopy() setReadyConditionFalse(&cr.Status.Conditions, reason, message, cr.Generation, resetTimer) cr.Status.StatusSummary = computeStatusSummary(cr.Spec, cr.Status, time.Now()) diff --git a/internal/scheduling/reservations/commitments/reservation_manager.go b/internal/scheduling/reservations/commitments/reservation_manager.go index cc83851e5..565082e15 100644 --- a/internal/scheduling/reservations/commitments/reservation_manager.go +++ b/internal/scheduling/reservations/commitments/reservation_manager.go @@ -161,7 +161,7 @@ func (m *ReservationManager) ApplyCommitmentState( memValue := res.Spec.Resources[hv1.ResourceMemory] deltaMemoryBytes += memValue.Value() - if err := m.Delete(ctx, &res); err != nil { + if err := client.IgnoreNotFound(m.Delete(ctx, &res)); err != nil { return result, fmt.Errorf("failed to delete reservation %s: %w", res.Name, err) } } else { @@ -186,7 +186,7 @@ func (m *ReservationManager) ApplyCommitmentState( "noHostsFoundSince", cond.LastTransitionTime.Time, "ttl", m.cfg.NoHostsFoundTTL, ) - if err := m.Delete(ctx, &res); err != nil { + if err := client.IgnoreNotFound(m.Delete(ctx, &res)); err != nil { return result, fmt.Errorf("failed to delete reservation %s: %w", res.Name, err) } result.Deleted++ @@ -229,7 +229,7 @@ func (m *ReservationManager) ApplyCommitmentState( memValue := reservationToDelete.Spec.Resources[hv1.ResourceMemory] deltaMemoryBytes += memValue.Value() - if err := m.Delete(ctx, reservationToDelete); err != nil { + if err := client.IgnoreNotFound(m.Delete(ctx, reservationToDelete)); err != nil { return result, fmt.Errorf("failed to delete reservation %s: %w", reservationToDelete.Name, err) } } From 5a36b826c8ea1f9c6c7ccc797293f75a731fc2db Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:52:16 +0000 Subject: [PATCH 04/20] Renovate: Update External dependencies (#1371) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Adoption](https://docs.renovatebot.com/merge-confidence/) | [Passing](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | Type | Update | |---|---|---|---|---|---|---|---| | [github.com/cobaltcore-dev/vm-operator](https://redirect.github.com/cobaltcore-dev/vm-operator) | `v0.1.2-0.20261014085525-20c781d5a4eb` → `v0.2.4` | ![age](https://developer.mend.io/api/mc/badges/age/go/github.com%2fcobaltcore-dev%2fvm-operator/v0.2.4?slim=true) | ![adoption](https://developer.mend.io/api/mc/badges/adoption/go/github.com%2fcobaltcore-dev%2fvm-operator/v0.2.4?slim=true) | ![passing](https://developer.mend.io/api/mc/badges/compatibility/go/github.com%2fcobaltcore-dev%2fvm-operator/v0.1.2-0.20261014085525-20c781d5a4eb/v0.2.4?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/go/github.com%2fcobaltcore-dev%2fvm-operator/v0.1.2-0.20261014085525-20c781d5a4eb/v0.2.4?slim=true) | require | minor | | [kube-prometheus-stack](https://redirect.github.com/prometheus-operator/kube-prometheus) ([source](https://redirect.github.com/prometheus-community/helm-charts)) | `92.7.0` → `92.9.0` | ![age](https://developer.mend.io/api/mc/badges/age/docker/ghcr.io%2fprometheus-community%2fcharts%2fkube-prometheus-stack/92.9.0?slim=true) | ![adoption](https://developer.mend.io/api/mc/badges/adoption/docker/ghcr.io%2fprometheus-community%2fcharts%2fkube-prometheus-stack/92.9.0?slim=true) | ![passing](https://developer.mend.io/api/mc/badges/compatibility/docker/ghcr.io%2fprometheus-community%2fcharts%2fkube-prometheus-stack/92.7.0/92.9.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/docker/ghcr.io%2fprometheus-community%2fcharts%2fkube-prometheus-stack/92.7.0/92.9.0?slim=true) | | minor | | [sigs.k8s.io/controller-tools](https://redirect.github.com/kubernetes-sigs/controller-tools) | `v0.22.0` → `v0.23.0` | ![age](https://developer.mend.io/api/mc/badges/age/go/sigs.k8s.io%2fcontroller-tools/v0.23.0?slim=true) | ![adoption](https://developer.mend.io/api/mc/badges/adoption/go/sigs.k8s.io%2fcontroller-tools/v0.23.0?slim=true) | ![passing](https://developer.mend.io/api/mc/badges/compatibility/go/sigs.k8s.io%2fcontroller-tools/v0.22.0/v0.23.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/go/sigs.k8s.io%2fcontroller-tools/v0.22.0/v0.23.0?slim=true) | | minor | --- ### Release Notes
cobaltcore-dev/vm-operator (github.com/cobaltcore-dev/vm-operator) ### [`v0.2.4`](https://redirect.github.com/cobaltcore-dev/vm-operator/releases/tag/v0.2.4) [Compare Source](https://redirect.github.com/cobaltcore-dev/vm-operator/compare/v0.2.3...v0.2.4) #### What's Changed - Bugfixes after initial deployment by [@​PhilippMatthes](https://redirect.github.com/PhilippMatthes) in [#​8](https://redirect.github.com/cobaltcore-dev/vm-operator/pull/8) **Full Changelog**: ### [`v0.2.3`](https://redirect.github.com/cobaltcore-dev/vm-operator/releases/tag/v0.2.3) [Compare Source](https://redirect.github.com/cobaltcore-dev/vm-operator/compare/v0.2.2...v0.2.3) **Full Changelog**: ### [`v0.2.2`](https://redirect.github.com/cobaltcore-dev/vm-operator/releases/tag/v0.2.2) [Compare Source](https://redirect.github.com/cobaltcore-dev/vm-operator/compare/v0.2.1...v0.2.2) **Full Changelog**: ### [`v0.2.1`](https://redirect.github.com/cobaltcore-dev/vm-operator/releases/tag/v0.2.1) [Compare Source](https://redirect.github.com/cobaltcore-dev/vm-operator/compare/v0.2.0...v0.2.1) **Full Changelog**:
prometheus-community/helm-charts (kube-prometheus-stack) ### [`v92.9.0`](https://redirect.github.com/prometheus-community/helm-charts/releases/tag/kube-prometheus-stack-92.9.0) [Compare Source](https://redirect.github.com/prometheus-community/helm-charts/compare/kube-prometheus-stack-92.8.0...kube-prometheus-stack-92.9.0) kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. ##### What's Changed - \[kube-prometheus-stack] Update grafana Docker tag to v14 by [@​renovate](https://redirect.github.com/renovate)\[bot] in [#​7380](https://redirect.github.com/prometheus-community/helm-charts/pull/7380) **Full Changelog**: ### [`v92.8.0`](https://redirect.github.com/prometheus-community/helm-charts/releases/tag/kube-prometheus-stack-92.8.0) [Compare Source](https://redirect.github.com/prometheus-community/helm-charts/compare/kube-prometheus-stack-92.7.0...kube-prometheus-stack-92.8.0) kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. ##### What's Changed - \[kube-prometheus-stack] Update grafana Docker tag to v13.7.0 by [@​renovate](https://redirect.github.com/renovate)\[bot] in [#​7379](https://redirect.github.com/prometheus-community/helm-charts/pull/7379) **Full Changelog**:
kubernetes-sigs/controller-tools (sigs.k8s.io/controller-tools) ### [`v0.23.0`](https://redirect.github.com/kubernetes-sigs/controller-tools/releases/tag/v0.23.0) [Compare Source](https://redirect.github.com/kubernetes-sigs/controller-tools/compare/v0.22.0...v0.23.0) This is an additional minor release independent of the Kubernetes release cadence to deliver [#​1474](https://redirect.github.com/kubernetes-sigs/controller-tools/pull/1474) because this bug fix is a breaking change so it was not possible to backport it. #### What's Changed - ✨ Add option to omit controller-gen version in generated output by [@​pedjak](https://redirect.github.com/pedjak) in [#​1437](https://redirect.github.com/kubernetes-sigs/controller-tools/pull/1437) - 🐛 Fix +listType marker not working on type alias fields by [@​sivchari](https://redirect.github.com/sivchari) in [#​1359](https://redirect.github.com/kubernetes-sigs/controller-tools/pull/1359) - 🐛 Inject min boundaries for +k8s:required on strings, maps, and slices by [@​lalitc375](https://redirect.github.com/lalitc375) in [#​1458](https://redirect.github.com/kubernetes-sigs/controller-tools/pull/1458) - 🐛 crd: support Kubernetes declarative validation bounds by [@​kannon92](https://redirect.github.com/kannon92) in [#​1474](https://redirect.github.com/kubernetes-sigs/controller-tools/pull/1474) #### Dependency bumps - 🌱 Bump github.com/onsi/gomega from 1.43.1 to 1.44.0 in the all-go-mod-patch-and-minor group by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1480](https://redirect.github.com/kubernetes-sigs/controller-tools/pull/1480) - 🌱 Bump golang.org/x/tools from 0.50.0 to 0.51.0 in the all-go-mod-patch-and-minor group by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1487](https://redirect.github.com/kubernetes-sigs/controller-tools/pull/1487) - 🌱 Bump golangci-lint to v2.14.0 and enable promlinter, reassign linters by [@​dongjiang1989](https://redirect.github.com/dongjiang1989) in [#​1488](https://redirect.github.com/kubernetes-sigs/controller-tools/pull/1488) - 🌱 Bump sigs.k8s.io/controller-runtime from v0.24.0 to v0.25.0 by [@​dongjiang1989](https://redirect.github.com/dongjiang1989) in [#​1476](https://redirect.github.com/kubernetes-sigs/controller-tools/pull/1476) - 🌱 Bump softprops/action-gh-release from 3.0.2 to 3.0.3 in the all-github-actions group by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1475](https://redirect.github.com/kubernetes-sigs/controller-tools/pull/1475) - 🌱 Bump the all-go-mod-patch-and-minor group across 1 directory with 3 updates by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1478](https://redirect.github.com/kubernetes-sigs/controller-tools/pull/1478) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://redirect.github.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Co-authored-by: Markus Wieland Co-authored-by: Claude Opus 4.7 --- Makefile | 2 +- go.mod | 2 +- go.sum | 8 ++------ helm/dev/cortex-prometheus-operator/Chart.yaml | 2 +- .../files/crds/cortex.cloud_committedresources.yaml | 2 +- .../cortex/files/crds/cortex.cloud_datasources.yaml | 2 +- .../library/cortex/files/crds/cortex.cloud_decisions.yaml | 2 +- .../cortex/files/crds/cortex.cloud_deschedulings.yaml | 2 +- .../files/crds/cortex.cloud_flavorgroupcapacities.yaml | 2 +- .../library/cortex/files/crds/cortex.cloud_histories.yaml | 2 +- .../cortex/files/crds/cortex.cloud_knowledges.yaml | 2 +- helm/library/cortex/files/crds/cortex.cloud_kpis.yaml | 2 +- .../library/cortex/files/crds/cortex.cloud_pipelines.yaml | 2 +- .../cortex/files/crds/cortex.cloud_projectquotas.yaml | 2 +- .../cortex/files/crds/cortex.cloud_reservations.yaml | 2 +- 15 files changed, 16 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 9fceacd7c..929e2b565 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen GOLANGCI_LINT = $(LOCALBIN)/golangci-lint GOTESTSUM = $(LOCALBIN)/gotestsum -CONTROLLER_TOOLS_VERSION ?= v0.22.0 +CONTROLLER_TOOLS_VERSION ?= v0.23.0 GOLANGCI_LINT_VERSION ?= v2.14.0 GOTESTSUM_VERSION ?= v1.13.0 diff --git a/go.mod b/go.mod index ed2a0b02a..6180c1721 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.27 require ( github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3 - github.com/cobaltcore-dev/vm-operator v0.1.2-0.20261014085525-20c781d5a4eb + github.com/cobaltcore-dev/vm-operator v0.2.4 github.com/go-gorp/gorp v2.2.0+incompatible github.com/go-logr/logr v1.4.4 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index af6ffdf0f..080ef3fdf 100644 --- a/go.sum +++ b/go.sum @@ -24,12 +24,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3 h1:2KjrD+LzU8pgvm1HEIEw5qlRF1gcbFp7Q/t9z+AnY4g= github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3/go.mod h1:DyQ6MO1TsC4kzeG3l8tnIXQVeA/PITm22RyitJhDaKI= -github.com/cobaltcore-dev/vm-operator v0.1.2-0.20261013143410-18c5ab1c4c9c h1:CB+9SiS9z+2cmWrHw519C6UU8SpzDezM6BktoshLlJE= -github.com/cobaltcore-dev/vm-operator v0.1.2-0.20261013143410-18c5ab1c4c9c/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4= -github.com/cobaltcore-dev/vm-operator v0.1.2-0.20261014064538-fa16f4c21e31 h1:CZpisES+VGjZrxzjHrsJ8n1OdsMb97yKRW8sGuyENe0= -github.com/cobaltcore-dev/vm-operator v0.1.2-0.20261014064538-fa16f4c21e31/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4= -github.com/cobaltcore-dev/vm-operator v0.1.2-0.20261014085525-20c781d5a4eb h1:Tg8nIIubpwT7oWbKfYfc9Y/xEToVk1nwraZyQ17WIJI= -github.com/cobaltcore-dev/vm-operator v0.1.2-0.20261014085525-20c781d5a4eb/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4= +github.com/cobaltcore-dev/vm-operator v0.2.4 h1:cCWjJ41+4bo1BSr/uZAXsTm+NZqwscaAUix5tDHmSU0= +github.com/cobaltcore-dev/vm-operator v0.2.4/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index e23ba5bf9..2ce45c4f5 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 92.7.0 + version: 92.9.0 diff --git a/helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml b/helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml index 568e5e02f..f14556733 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_committedresources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.22.0 + controller-gen.kubebuilder.io/version: v0.23.0 name: committedresources.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_datasources.yaml b/helm/library/cortex/files/crds/cortex.cloud_datasources.yaml index e949d0b9e..a90199d69 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_datasources.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_datasources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.22.0 + controller-gen.kubebuilder.io/version: v0.23.0 name: datasources.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml b/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml index 2d80182e0..a9fe3c78e 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.22.0 + controller-gen.kubebuilder.io/version: v0.23.0 name: decisions.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_deschedulings.yaml b/helm/library/cortex/files/crds/cortex.cloud_deschedulings.yaml index db0807f5a..9c41d2adf 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_deschedulings.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_deschedulings.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.22.0 + controller-gen.kubebuilder.io/version: v0.23.0 name: deschedulings.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml index a4244e7b9..cf860e731 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.22.0 + controller-gen.kubebuilder.io/version: v0.23.0 name: flavorgroupcapacities.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_histories.yaml b/helm/library/cortex/files/crds/cortex.cloud_histories.yaml index 3818ecdbf..5d72687b7 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_histories.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_histories.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.22.0 + controller-gen.kubebuilder.io/version: v0.23.0 name: histories.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_knowledges.yaml b/helm/library/cortex/files/crds/cortex.cloud_knowledges.yaml index a42984003..c8b20984d 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_knowledges.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_knowledges.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.22.0 + controller-gen.kubebuilder.io/version: v0.23.0 name: knowledges.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_kpis.yaml b/helm/library/cortex/files/crds/cortex.cloud_kpis.yaml index 68ec754d4..978f0ee5e 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_kpis.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_kpis.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.22.0 + controller-gen.kubebuilder.io/version: v0.23.0 name: kpis.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml b/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml index 0262ea445..21a69b62f 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_pipelines.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.22.0 + controller-gen.kubebuilder.io/version: v0.23.0 name: pipelines.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml b/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml index c48ae2d63..01fd59813 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_projectquotas.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.22.0 + controller-gen.kubebuilder.io/version: v0.23.0 name: projectquotas.cortex.cloud spec: group: cortex.cloud diff --git a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml index 4cd6101e6..efd4efb56 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.22.0 + controller-gen.kubebuilder.io/version: v0.23.0 name: reservations.cortex.cloud spec: group: cortex.cloud From ebdf9056cb3a3007ca9de2b260b82f4159c39dbe Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Fri, 16 Oct 2026 12:27:52 +0200 Subject: [PATCH 05/20] Populate cortex intent and explanation on VM spec (#1373) Bump vm-operator to v0.2.8 and set the new CortexSpec Explanation and Intent fields when writing the scheduling result. The intent lookup is moved earlier so it can feed both the spec fields and the status condition. --------- Signed-off-by: Philipp Matthes --- go.mod | 2 +- go.sum | 4 ++-- internal/scheduling/nova/vm_handler.go | 14 ++++++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 6180c1721..d94ba7649 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.27 require ( github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3 - github.com/cobaltcore-dev/vm-operator v0.2.4 + github.com/cobaltcore-dev/vm-operator v0.2.8 github.com/go-gorp/gorp v2.2.0+incompatible github.com/go-logr/logr v1.4.4 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 080ef3fdf..c44ebea7d 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3 h1:2KjrD+LzU8pgvm1HEIEw5qlRF1gcbFp7Q/t9z+AnY4g= github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3/go.mod h1:DyQ6MO1TsC4kzeG3l8tnIXQVeA/PITm22RyitJhDaKI= -github.com/cobaltcore-dev/vm-operator v0.2.4 h1:cCWjJ41+4bo1BSr/uZAXsTm+NZqwscaAUix5tDHmSU0= -github.com/cobaltcore-dev/vm-operator v0.2.4/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4= +github.com/cobaltcore-dev/vm-operator v0.2.8 h1:coH9Lptqxq6/M89wrfwUSJch30EYoZopKCrtasjUb58= +github.com/cobaltcore-dev/vm-operator v0.2.8/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= diff --git a/internal/scheduling/nova/vm_handler.go b/internal/scheduling/nova/vm_handler.go index 4b48fedd2..25ee32953 100644 --- a/internal/scheduling/nova/vm_handler.go +++ b/internal/scheduling/nova/vm_handler.go @@ -136,6 +136,12 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.D } } + // Needed to populate the Cortex intent field and the status condition. + intent, err := request.GetIntent() + if err != nil { + return fmt.Errorf("failed to get intent: %w", err) + } + // Write the cortex spec (our scheduling result). vm.Spec.Cortex = &vmv1alpha1.CortexSpec{} for _, host := range decision.Status.Result.OrderedHosts { @@ -145,15 +151,13 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.D Name: host, }) } + vm.Spec.Cortex.Explanation = decision.Status.Explanation + vm.Spec.Cortex.Intent = string(intent) // Write the ConditionTypeScheduled condition to indicate that the VM has // been successfully scheduled. For that we need to extract the scheduling // intent from request and verify that the operation var reason string - intent, err := request.GetIntent() - if err != nil { - return fmt.Errorf("failed to get intent: %w", err) - } switch intent { case api.LiveMigrationIntent: reason = vmv1alpha1.ConditionReasonScheduledLiveMigration @@ -221,6 +225,8 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.D vmInCluster.Spec.Cortex = &vmv1alpha1.CortexSpec{} } vmInCluster.Spec.Cortex.HostCandidates = vm.Spec.Cortex.HostCandidates + vmInCluster.Spec.Cortex.Explanation = vm.Spec.Cortex.Explanation + vmInCluster.Spec.Cortex.Intent = vm.Spec.Cortex.Intent if err := h.Update(ctx, vmInCluster); err != nil { return fmt.Errorf("failed to update VM: %w", err) } From d711e997628db7f6b2f5c244666558bcb5188577 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Fri, 16 Oct 2026 14:45:11 +0200 Subject: [PATCH 06/20] Fix scheduling intent in decision being set to Unknown always (#1374) Fixes a bug that lead to the virtual machine resources never being generated. Signed-off-by: Philipp Matthes --- internal/scheduling/nova/external_scheduler_api.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/scheduling/nova/external_scheduler_api.go b/internal/scheduling/nova/external_scheduler_api.go index 30f65b2aa..859b6ced0 100644 --- a/internal/scheduling/nova/external_scheduler_api.go +++ b/internal/scheduling/nova/external_scheduler_api.go @@ -255,6 +255,13 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req logger.Info("inferred pipeline name", "pipeline", requestData.Pipeline) } + intent, err := requestData.GetIntent() + if err != nil { + // Just log this in case the intent cannot be determined, but continue processing. + logger.Error("failed to get intent", "error", err) + intent = v1alpha1.SchedulingIntentUnknown + } + decision := &v1alpha1.Decision{ TypeMeta: metav1.TypeMeta{ Kind: "Decision", @@ -270,7 +277,7 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req }, ResourceID: requestData.Spec.Data.InstanceUUID, NovaRaw: &raw, - Intent: v1alpha1.SchedulingIntentUnknown, + Intent: intent, }, } ctx := r.Context() @@ -295,8 +302,7 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req } // This is a hack to address the problem that Nova only uses the first host in hosts for evacuation requests. // Only for evacuation we shuffle the first k hosts to ensure that we do not get stuck on a single host - intent, err := requestData.GetIntent() - if err == nil && intent == api.EvacuateIntent { + if intent == api.EvacuateIntent { hosts = shuffleTopHosts(hosts, httpAPI.config.EvacuationShuffleK) } // Ensure we always return an array, never null. Nova's schema requires From a1856d717711713bbcfde901209e3a3aec072444 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Fri, 16 Oct 2026 15:20:09 +0200 Subject: [PATCH 07/20] Default scheduling intent to create for nova api (#1376) Default the intent to api.CreateIntent instead of Unknown when it cannot be determined, and remove the redundant intent derivation in the pipeline controller that would otherwise override it with Unknown. --------- Signed-off-by: Philipp Matthes --- internal/scheduling/nova/external_scheduler_api.go | 4 ++-- .../scheduling/nova/filter_weigher_pipeline_controller.go | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/internal/scheduling/nova/external_scheduler_api.go b/internal/scheduling/nova/external_scheduler_api.go index 859b6ced0..e4e68d27a 100644 --- a/internal/scheduling/nova/external_scheduler_api.go +++ b/internal/scheduling/nova/external_scheduler_api.go @@ -258,8 +258,8 @@ func (httpAPI *httpAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Req intent, err := requestData.GetIntent() if err != nil { // Just log this in case the intent cannot be determined, but continue processing. - logger.Error("failed to get intent", "error", err) - intent = v1alpha1.SchedulingIntentUnknown + logger.Info("found no request intent (intended for initial placements)") + intent = api.CreateIntent } decision := &v1alpha1.Decision{ diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller.go b/internal/scheduling/nova/filter_weigher_pipeline_controller.go index f63eed57b..451298928 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller.go @@ -180,13 +180,6 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision return nil, err } - if intent, err := request.GetIntent(); err != nil { - log.Error(err, "failed to get intent from nova request, using Unknown") - decision.Spec.Intent = v1alpha1.SchedulingIntentUnknown - } else { - decision.Spec.Intent = intent - } - // If necessary gather all placement candidates before filtering. // This will override the hosts and weights in the nova request. pipelineConf, ok := c.GetPipelineConfig(decision.Spec.PipelineRef.Name) From 200d896c4143d9fe05b8f0cf63ef8f6e035d47cb Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Fri, 16 Oct 2026 16:35:31 +0200 Subject: [PATCH 08/20] Fix VM status condition being wiped on create path h.Create writes the apiserver response (with an empty status subresource) back into the vm object, overwriting the Scheduled condition set earlier. The subsequent Status().Update then persisted an empty status. Snapshot the conditions before Create and restore them before the status update. Signed-off-by: Philipp Matthes --- internal/scheduling/nova/vm_handler.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/scheduling/nova/vm_handler.go b/internal/scheduling/nova/vm_handler.go index 25ee32953..957b50503 100644 --- a/internal/scheduling/nova/vm_handler.go +++ b/internal/scheduling/nova/vm_handler.go @@ -186,9 +186,14 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.D Name: vm.Name, }, vmInCluster) if err != nil && apierrors.IsNotFound(err) { + // Create ignores the status subresource and writes the server response + // (with an empty status) back into vm, so snapshot the conditions and + // restore them before persisting the status. + conditions := vm.Status.Conditions if err := h.Create(ctx, vm); err != nil { return fmt.Errorf("failed to create VM: %w", err) } + vm.Status.Conditions = conditions if err := h.Status().Update(ctx, vm); err != nil { return fmt.Errorf("failed to update VM status: %w", err) } From 7268037088bb27fa6042aaf9d1b5ef27287071b3 Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Fri, 16 Oct 2026 16:46:23 +0200 Subject: [PATCH 09/20] chore: remove history crd (#1375) Explanation is now added to the vm crd. - adjusted the docs - removed history crd - removed history client - added plain explanation generation Signed-off-by: Markus Wieland --- api/scheduling/options.go | 9 +- api/scheduling/options_test.go | 7 +- api/v1alpha1/decision_types.go | 3 +- api/v1alpha1/history_types.go | 128 ---- api/v1alpha1/zz_generated.deepcopy.go | 158 ----- cmd/manager/main.go | 47 -- .../02-architecture-at-a-glance.md | 5 +- .../02-nova-smart-scheduling.md | 33 +- .../03-manila-smart-scheduling.md | 2 +- .../04-cinder-smart-scheduling.md | 4 +- .../05-pod-gang-scheduling.md | 2 +- .../06-ironcore-machine-scheduling.md | 2 +- .../01-multicluster-client.md | 67 +- helm/bundles/cortex-cinder/values.yaml | 4 - helm/bundles/cortex-ironcore/values.yaml | 2 - helm/bundles/cortex-manila/values.yaml | 4 - .../cortex-nova/templates/kpis_kvm.yaml | 1 - .../cortex-nova/templates/pipelines_kvm.yaml | 337 ---------- helm/bundles/cortex-nova/values.yaml | 3 - helm/bundles/cortex-pods/values.yaml | 2 - .../files/crds/cortex.cloud_decisions.yaml | 3 +- .../files/crds/cortex.cloud_histories.yaml | 303 --------- .../filter_weigher_pipeline_controller.go | 13 +- ...filter_weigher_pipeline_controller_test.go | 63 +- internal/scheduling/cinder/history_cleanup.go | 130 ---- .../scheduling/cinder/history_cleanup_test.go | 388 ----------- internal/scheduling/lib/explanation.go | 108 +++ internal/scheduling/lib/explanation_test.go | 169 +++++ internal/scheduling/lib/history_client.go | 308 --------- .../scheduling/lib/history_client_test.go | 631 ------------------ .../scheduling/lib/pipeline_controller.go | 2 - .../filter_weigher_pipeline_controller.go | 24 +- ...filter_weigher_pipeline_controller_test.go | 105 +-- .../filter_weigher_pipeline_controller.go | 13 +- ...filter_weigher_pipeline_controller_test.go | 63 +- internal/scheduling/manila/history_cleanup.go | 134 ---- .../scheduling/manila/history_cleanup_test.go | 438 ------------ internal/scheduling/nova/cr_metrics_test.go | 3 +- .../filter_weigher_pipeline_controller.go | 33 +- ...filter_weigher_pipeline_controller_test.go | 86 +-- internal/scheduling/nova/history_cleanup.go | 147 ---- .../scheduling/nova/history_cleanup_test.go | 440 ------------ internal/scheduling/nova/integration_test.go | 1 - .../filter_weigher_pipeline_controller.go | 25 +- ...filter_weigher_pipeline_controller_test.go | 78 +-- .../reservations/capacity/controller.go | 1 - .../commitments/reservation_controller.go | 1 - .../reservations/failover/integration_test.go | 2 - .../failover/reservation_scheduling.go | 3 +- pkg/multicluster/recorder_test.go | 18 +- pkg/multicluster/routers.go | 48 -- pkg/multicluster/routers_test.go | 109 --- 52 files changed, 441 insertions(+), 4269 deletions(-) delete mode 100644 api/v1alpha1/history_types.go delete mode 100644 helm/library/cortex/files/crds/cortex.cloud_histories.yaml delete mode 100644 internal/scheduling/cinder/history_cleanup.go delete mode 100644 internal/scheduling/cinder/history_cleanup_test.go create mode 100644 internal/scheduling/lib/explanation.go create mode 100644 internal/scheduling/lib/explanation_test.go delete mode 100644 internal/scheduling/lib/history_client.go delete mode 100644 internal/scheduling/lib/history_client_test.go delete mode 100644 internal/scheduling/manila/history_cleanup.go delete mode 100644 internal/scheduling/manila/history_cleanup_test.go delete mode 100644 internal/scheduling/nova/history_cleanup.go delete mode 100644 internal/scheduling/nova/history_cleanup_test.go diff --git a/api/scheduling/options.go b/api/scheduling/options.go index 1d5991cc4..2cccd1d54 100644 --- a/api/scheduling/options.go +++ b/api/scheduling/options.go @@ -13,8 +13,8 @@ import ( // These are distinct from per-step YAML options (FilterWeigherPipelineStepOpts), // which are static and set when the pipeline is initialized. type Options struct { - // ReadOnly means the pipeline run does not modify shared scheduling state (reservations, - // history, ...). Cortex may run read-only runs concurrently. + // ReadOnly means the pipeline run does not modify shared scheduling state (reservations, ...). + // Cortex may run read-only runs concurrently. ReadOnly bool `json:"read_only,omitempty"` // AssumeEmptyHosts ignores running instances on hosts, considering them as empty. @@ -27,8 +27,6 @@ type Options struct { // MaxCandidates limits the number of candidates (hosts) returned after weighing. 0 means no limit. MaxCandidates int `json:"max_candidates,omitempty"` - // SkipHistory skips recording the placement decision in placement history. - SkipHistory bool `json:"skip_history,omitempty"` // SkipInflight skips creating pessimistic blocking reservations for returned candidates. SkipInflight bool `json:"skip_inflight,omitempty"` // SkipCommittedResourceTracking skips writing the placed VM UUID into the matching @@ -39,9 +37,6 @@ type Options struct { // Validate checks for mutually exclusive or inconsistent option combinations. func (o Options) Validate() error { - if o.ReadOnly && !o.SkipHistory { - return errors.New("read-only runs must not write scheduling history: set SkipHistory=true") - } if o.ReadOnly && !o.SkipInflight { return errors.New("read-only runs cannot create inflight reservations") } diff --git a/api/scheduling/options_test.go b/api/scheduling/options_test.go index fd1ccfee2..681798191 100644 --- a/api/scheduling/options_test.go +++ b/api/scheduling/options_test.go @@ -12,10 +12,9 @@ func TestOptions_Validate(t *testing.T) { wantErr bool }{ {"zero value is valid", Options{}, false}, - {"read-only run, skipping all writes", Options{ReadOnly: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}, false}, - {"ReadOnly without SkipHistory is invalid", Options{ReadOnly: true}, true}, - {"ReadOnly without SkipInflight is invalid", Options{ReadOnly: true, SkipHistory: true}, true}, - {"ReadOnly without SkipCommittedResourceTracking is invalid", Options{ReadOnly: true, SkipHistory: true, SkipInflight: true}, true}, + {"read-only run, skipping all writes", Options{ReadOnly: true, SkipInflight: true, SkipCommittedResourceTracking: true}, false}, + {"ReadOnly without SkipInflight is invalid", Options{ReadOnly: true}, true}, + {"ReadOnly without SkipCommittedResourceTracking is invalid", Options{ReadOnly: true, SkipInflight: true}, true}, } for _, tt := range tests { diff --git a/api/v1alpha1/decision_types.go b/api/v1alpha1/decision_types.go index 0dbd6418d..fb1e07ca9 100644 --- a/api/v1alpha1/decision_types.go +++ b/api/v1alpha1/decision_types.go @@ -125,7 +125,8 @@ type DecisionStatus struct { // Currently the Decision CRD is an in-memory scheduling object used by the external scheduler API // and filter-weigher pipelines to compute a placement result. It is currently NOT persisted -// to etcd — the scheduling outcome is recorded in the History CRD instead. +// to etcd — the scheduling outcome (including a human-readable explanation) is written onto the +// scheduled resource instead, e.g. the nova VM's spec.cortex fields. // The long-term shape of this CRD is still under discussion; until that is // settled the Decision serves only as a transient carrier within a single // scheduling run. diff --git a/api/v1alpha1/history_types.go b/api/v1alpha1/history_types.go deleted file mode 100644 index 47a37dec6..000000000 --- a/api/v1alpha1/history_types.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package v1alpha1 - -import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -type SchedulingHistoryEntry struct { - // The timestamp of when the decision was made. - Timestamp metav1.Time `json:"timestamp"` - // The pipeline that was used for the decision. - PipelineRef corev1.ObjectReference `json:"pipelineRef"` - // The intent of the decision (e.g., initial scheduling, rescheduling, etc.). - Intent SchedulingIntent `json:"intent"` - // The top hosts ordered by score for the decision (limited to 3). - // This is not a complete list of all candidates — only the highest-ranked - // hosts are retained to keep the history compact. - // +kubebuilder:validation:Optional - // +kubebuilder:validation:MaxItems=3 - OrderedHosts []string `json:"orderedHosts,omitempty"` - // Whether the scheduling decision was successful. - // +kubebuilder:validation:Optional - Successful bool `json:"successful"` -} - -const ( - // The scheduling decision is ready (a host was successfully selected). - HistoryConditionReady = "Ready" - // The scheduling decision selected a target host. - HistoryReasonSchedulingSucceeded = "SchedulingSucceeded" - // The pipeline run failed before a host could be selected. - HistoryReasonPipelineRunFailed = "PipelineRunFailed" - // The pipeline completed but no suitable host was found. - HistoryReasonNoHostFound = "NoHostFound" -) - -type HistorySpec struct { - // The scheduling domain this object with the history belongs to. - SchedulingDomain SchedulingDomain `json:"schedulingDomain"` - // The resource ID this history belongs to (e.g., the UUID of a nova instance). - ResourceID string `json:"resourceID"` - // The availability zone of the resource, if known. Only set for scheduling - // domains that provide AZ information (e.g., Nova). - // +kubebuilder:validation:Optional - AvailabilityZone *string `json:"availabilityZone,omitempty"` -} - -// CurrentDecision holds the full context of the most recent scheduling -// decision. When a new decision arrives the previous CurrentDecision is -// compacted into a SchedulingHistoryEntry and appended to History. -type CurrentDecision struct { - // The timestamp of when the decision was made. - Timestamp metav1.Time `json:"timestamp"` - // The pipeline that was used for the decision. - PipelineRef corev1.ObjectReference `json:"pipelineRef"` - // The intent of the decision (e.g., initial scheduling, rescheduling, etc.). - Intent SchedulingIntent `json:"intent"` - // Whether the scheduling decision was successful. - Successful bool `json:"successful"` - // The target host selected for the resource. nil when no host was found. - // +kubebuilder:validation:Optional - TargetHost *string `json:"targetHost,omitempty"` - // A human-readable explanation of the scheduling decision. - // +kubebuilder:validation:Optional - Explanation string `json:"explanation,omitempty"` - // The top hosts ordered by score (limited to 3). - // +kubebuilder:validation:Optional - // +kubebuilder:validation:MaxItems=3 - OrderedHosts []string `json:"orderedHosts,omitempty"` -} - -type HistoryStatus struct { - // Current represents the latest scheduling decision with full context. - // +kubebuilder:validation:Optional - Current CurrentDecision `json:"current,omitempty"` - // History of past scheduling decisions (limited to last 10). - // +kubebuilder:validation:Optional - History []SchedulingHistoryEntry `json:"history,omitempty"` - - // Conditions represent the latest available observations of the history's state. - // +kubebuilder:validation:Optional - Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Cluster -// +kubebuilder:printcolumn:name="Domain",type="string",JSONPath=".spec.schedulingDomain" -// +kubebuilder:printcolumn:name="Resource ID",type="string",JSONPath=".spec.resourceID" -// +kubebuilder:printcolumn:name="AZ",type="string",JSONPath=".spec.availabilityZone" -// +kubebuilder:printcolumn:name="Target Host",type="string",JSONPath=".status.current.targetHost" -// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].reason" -// +kubebuilder:printcolumn:name="Last Scheduled",type="date",JSONPath=".status.current.timestamp" -// +kubebuilder:printcolumn:name="Created",type="date",JSONPath=".metadata.creationTimestamp" - -// The history is a CRD that provides a record of past scheduling decisions for a given resource (e.g., a nova instance). -// A new history entry is created for each scheduling decision, and the most recent decision is stored in the status.current field. The history is capped at 10 entries to prevent unbounded growth. -// This CRD is designed to be used by an operations team to troubleshoot scheduling decisions and understand the context around why a particular host was selected (or not selected) for a resource. -type History struct { - metav1.TypeMeta `json:",inline"` - - // Standard object metadata. - // +optional - metav1.ObjectMeta `json:"metadata,omitempty"` - - // Spec defines the desired state of History. - // +required - Spec HistorySpec `json:"spec"` - // Status defines the observed state of History. - // +optional - Status HistoryStatus `json:"status,omitempty"` -} - -// +kubebuilder:object:root=true - -// HistoryList contains a list of History -type HistoryList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []History `json:"items"` -} - -func init() { - SchemeBuilder.Register(&History{}, &HistoryList{}) -} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2497e98db..01e9ec113 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -240,33 +240,6 @@ func (in *CommittedResourceStatus) DeepCopy() *CommittedResourceStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CurrentDecision) DeepCopyInto(out *CurrentDecision) { - *out = *in - in.Timestamp.DeepCopyInto(&out.Timestamp) - out.PipelineRef = in.PipelineRef - if in.TargetHost != nil { - in, out := &in.TargetHost, &out.TargetHost - *out = new(string) - **out = **in - } - if in.OrderedHosts != nil { - in, out := &in.OrderedHosts, &out.OrderedHosts - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CurrentDecision. -func (in *CurrentDecision) DeepCopy() *CurrentDecision { - if in == nil { - return nil - } - out := new(CurrentDecision) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Datasource) DeepCopyInto(out *Datasource) { *out = *in @@ -923,115 +896,6 @@ func (in *FlavorGroupCapacityStatus) DeepCopy() *FlavorGroupCapacityStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *History) DeepCopyInto(out *History) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new History. -func (in *History) DeepCopy() *History { - if in == nil { - return nil - } - out := new(History) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *History) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HistoryList) DeepCopyInto(out *HistoryList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]History, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HistoryList. -func (in *HistoryList) DeepCopy() *HistoryList { - if in == nil { - return nil - } - out := new(HistoryList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *HistoryList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HistorySpec) DeepCopyInto(out *HistorySpec) { - *out = *in - if in.AvailabilityZone != nil { - in, out := &in.AvailabilityZone, &out.AvailabilityZone - *out = new(string) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HistorySpec. -func (in *HistorySpec) DeepCopy() *HistorySpec { - if in == nil { - return nil - } - out := new(HistorySpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HistoryStatus) DeepCopyInto(out *HistoryStatus) { - *out = *in - in.Current.DeepCopyInto(&out.Current) - if in.History != nil { - in, out := &in.History, &out.History - *out = make([]SchedulingHistoryEntry, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HistoryStatus. -func (in *HistoryStatus) DeepCopy() *HistoryStatus { - if in == nil { - return nil - } - out := new(HistoryStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IdentityDatasource) DeepCopyInto(out *IdentityDatasource) { *out = *in @@ -1909,28 +1773,6 @@ func (in *ReservationStatus) DeepCopy() *ReservationStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SchedulingHistoryEntry) DeepCopyInto(out *SchedulingHistoryEntry) { - *out = *in - in.Timestamp.DeepCopyInto(&out.Timestamp) - out.PipelineRef = in.PipelineRef - if in.OrderedHosts != nil { - in, out := &in.OrderedHosts, &out.OrderedHosts - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulingHistoryEntry. -func (in *SchedulingHistoryEntry) DeepCopy() *SchedulingHistoryEntry { - if in == nil { - return nil - } - out := new(SchedulingHistoryEntry) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StepResult) DeepCopyInto(out *StepResult) { *out = *in diff --git a/cmd/manager/main.go b/cmd/manager/main.go index e9bcfc25f..07e69f1c0 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -15,7 +15,6 @@ import ( "path/filepath" "slices" "strings" - "time" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -383,7 +382,6 @@ func main() { {Group: "kvm.cloud.sap", Version: "v1", Kind: "Hypervisor"}: multicluster.HypervisorResourceRouter{}, {Group: "cobaltcore.cloud", Version: "v1alpha1", Kind: "VirtualMachine"}: vmopmcl.VirtualMachineRouter{}, {Group: "cortex.cloud", Version: "v1alpha1", Kind: "Reservation"}: multicluster.ReservationsResourceRouter{}, - {Group: "cortex.cloud", Version: "v1alpha1", Kind: "History"}: multicluster.HistoryResourceRouter{}, {Group: "cortex.cloud", Version: "v1alpha1", Kind: "CommittedResource"}: multicluster.CommittedResourceRouter{}, {Group: "cortex.cloud", Version: "v1alpha1", Kind: "FlavorGroupCapacity"}: multicluster.FlavorGroupCapacityResourceRouter{}, {Group: "cortex.cloud", Version: "v1alpha1", Kind: "ProjectQuota"}: multicluster.ProjectQuotaResourceRouter{}, @@ -960,51 +958,6 @@ func main() { os.Exit(1) } } - if slices.Contains(mainConfig.EnabledTasks, "nova-history-cleanup-task") { - setupLog.Info("starting nova history cleanup task") - historyCleanupConfig := conf.GetConfigOrDie[nova.HistoryCleanupConfig]() - if err := (&task.Runner{ - Client: multiclusterClient, - Interval: time.Hour, - Name: "nova-history-cleanup-task", - Run: func(ctx context.Context) error { - return nova.HistoryCleanup(ctx, multiclusterClient, historyCleanupConfig) - }, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to add nova history cleanup task to manager") - os.Exit(1) - } - } - if slices.Contains(mainConfig.EnabledTasks, "manila-history-cleanup-task") { - setupLog.Info("starting manila history cleanup task") - historyCleanupConfig := conf.GetConfigOrDie[manila.HistoryCleanupConfig]() - if err := (&task.Runner{ - Client: multiclusterClient, - Interval: time.Hour, - Name: "manila-history-cleanup-task", - Run: func(ctx context.Context) error { - return manila.HistoryCleanup(ctx, multiclusterClient, historyCleanupConfig) - }, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to add manila history cleanup task to manager") - os.Exit(1) - } - } - if slices.Contains(mainConfig.EnabledTasks, "cinder-history-cleanup-task") { - setupLog.Info("starting cinder history cleanup task") - historyCleanupConfig := conf.GetConfigOrDie[cinder.HistoryCleanupConfig]() - if err := (&task.Runner{ - Client: multiclusterClient, - Interval: time.Hour, - Name: "cinder-history-cleanup-task", - Run: func(ctx context.Context) error { - return cinder.HistoryCleanup(ctx, multiclusterClient, historyCleanupConfig) - }, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to add cinder history cleanup task to manager") - os.Exit(1) - } - } signalCtx := ctrl.SetupSignalHandler() diff --git a/docs/01-getting-started/02-architecture-at-a-glance.md b/docs/01-getting-started/02-architecture-at-a-glance.md index c7e7cde99..a2fc77c1b 100644 --- a/docs/01-getting-started/02-architecture-at-a-glance.md +++ b/docs/01-getting-started/02-architecture-at-a-glance.md @@ -105,8 +105,7 @@ flowchart LR - **Reservation** models capacity that is promised but not yet consumed, so a pipeline treats it as unavailable. -That is the whole picture at a glance. Cortex owns eleven kinds in all — besides the six above, -`History` records past decisions, and `CommittedResource`, `ProjectQuota`, and `FlavorGroupCapacity` +That is the whole picture at a glance. Cortex owns ten kinds in all — besides the ones above, `CommittedResource`, `ProjectQuota`, and `FlavorGroupCapacity` round out the reservation-and-quota family, covered in [Chapter 3](../03-reservations-and-inventory/readme.md). Every field of every kind is defined on the Go types under `api/v1alpha1/*_types.go`; the generated CRD manifests are under @@ -116,7 +115,7 @@ Cortex also *consumes but does not own* external kinds: `Hypervisor` (`kvm.cloud `Machine`, `MachinePool`, and `MachineClass` (`compute.ironcore.dev/v1alpha1`). Because the kinds are cluster-scoped but a cluster may run more than one Cortex deployment, the -domain-owned kinds (`Datasource`, `Knowledge`, `KPI`, `Pipeline`, `Decision`, `History`, and the +domain-owned kinds (`Datasource`, `Knowledge`, `KPI`, `Pipeline`, `Decision`, and the reservation kinds) carry a **`spec.schedulingDomain`** field — one of `nova`, `cinder`, `manila`, `machines`, or `pods`. Each domain controller watches with a predicate that ignores any object whose `schedulingDomain` does not match its own, so a `cortex-nova` and a `cortex-cinder` deployment can diff --git a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md index 2065d231a..341f2b107 100644 --- a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md +++ b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md @@ -137,9 +137,9 @@ knowledge feature) exceeds a threshold over an observed window. The flow: issue real live-migrations via the Nova client). 4. A **cleanup** controller TTL-expires old `Descheduling` resources (and runs once on startup). -## Inspecting decisions, descheduling, and history +## Inspecting decisions and descheduling -Three cluster-scoped CRDs record what the pipelines did — inspect them with `kubectl` (no namespace, +Two cluster-scoped CRDs record what the pipelines did — inspect them with `kubectl` (no namespace, Cortex kinds are cluster-scoped). Each request that hits the external scheduler produces a **`Decision`**: ```bash @@ -152,7 +152,8 @@ nova a1b2c3… 1 2m kvm-general-purpose-load-balancing node ``` `TargetHost` (`.status.result.targetHost`) is what Cortex recommended and `Pipeline` names the pipeline -that ran; `-o yaml` shows the full filter/weigher trace. The detector pipeline emits **`Descheduling`** +that ran; `-o yaml` shows the full filter/weigher trace, and `.status.explanation` carries a +human-readable summary of why that host won. The detector pipeline emits **`Descheduling`** resources for VMs it wants to move: ```bash @@ -164,26 +165,16 @@ PREVIOUS HOST NEW HOST CREATED REASON(S) READY node-042 node-017 30s avoid_high_steal_pct True ``` -And **`History`** keeps the rolling record of past scheduling for a resource, so you can see where a VM has -been placed over time: - -```bash -kubectl get histories -``` - -``` -DOMAIN RESOURCE ID AZ TARGET HOST STATUS LAST SCHEDULED CREATED -nova a1b2c3… az-a node-017 Successful 30s 3d -``` - -The columns come from each resource's status; the field-by-field definitions live on the `Decision`, -`Descheduling`, and `History` Go types in `api/v1alpha1/`. +The columns come from each resource's status; the field-by-field definitions live on the `Decision` +and `Descheduling` Go types in `api/v1alpha1/`. For VM placements the same human-readable explanation +is also written onto the scheduled VM's `spec.cortex.explanation` (the vm-operator `VirtualMachine`), +so operators can trace a placement from the VM itself. Nova exercises the full arc: an HTTP request drives a filter-weigher pipeline whose weighers read -knowledge features and respect reservations, the result is a `Decision`, committed slots are recorded, and -a scheduled detector emits `Descheduling` recommendations recorded as `History`. Every other domain in -this chapter is a smaller version of this same shape — starting with the storage domains, which are -HTTP-driven like Nova but ship far fewer steps. +knowledge features and respect reservations, the result is a `Decision` (with its explanation recorded +on the scheduled VM), committed slots are recorded, and a scheduled detector emits `Descheduling` +recommendations. Every other domain in this chapter is a smaller version of this same shape — +starting with the storage domains, which are HTTP-driven like Nova but ship far fewer steps. ## Next diff --git a/docs/02-external-scheduler-api/03-manila-smart-scheduling.md b/docs/02-external-scheduler-api/03-manila-smart-scheduling.md index e3bb3327a..0ee8f6087 100644 --- a/docs/02-external-scheduler-api/03-manila-smart-scheduling.md +++ b/docs/02-external-scheduler-api/03-manila-smart-scheduling.md @@ -41,7 +41,7 @@ CPU load, respecting Manila's incoming weights." > [Extend Cortex](07-extending-cortex.md) — implement the step, self-register it, and > reference it from the `manila-external-scheduler` pipeline. -Manila is a second consumer of the same HTTP external-scheduler pattern and the same `Decision`/`History` +Manila is a second consumer of the same HTTP external-scheduler pattern and the same `Decision` machinery as Nova, proving the "write the engine once, reuse per domain" design from [Architecture at a glance](../01-getting-started/02-architecture-at-a-glance.md). Its one weigher reads a NetApp-usage feature from the [knowledge database](../04-knowledge-database/readme.md). Cinder, next, is diff --git a/docs/02-external-scheduler-api/04-cinder-smart-scheduling.md b/docs/02-external-scheduler-api/04-cinder-smart-scheduling.md index 990013d85..03ed53cd2 100644 --- a/docs/02-external-scheduler-api/04-cinder-smart-scheduling.md +++ b/docs/02-external-scheduler-api/04-cinder-smart-scheduling.md @@ -31,12 +31,12 @@ order (see the input-weight handling in [The scheduling engine](01-the-schedulin > [!NOTE] > An empty step set means Cinder scheduling is a *pass-through* today — Cortex is wired in and producing -> `Decision`/`History` records, but not yet altering Cinder's ordering. The plumbing is in place so that +> `Decision` records, but not yet altering Cinder's ordering. The plumbing is in place so that > adding block-storage-specific logic later is only a matter of implementing and registering steps > ([Extend Cortex](07-extending-cortex.md)); the integration does not have to change. Cinder shows the integration cost of a new domain in its purest form: a route, a default pipeline name, -and the shared `Decision`/`History` machinery — with the actual placement intelligence added later as +and the shared `Decision` machinery — with the actual placement intelligence added later as plugins. It is the same engine as Nova and Manila with the plugin set empty. The next two domains break from the HTTP pattern: pods and IronCore machines are scheduled by *watching* Kubernetes resources rather than answering an HTTP call. diff --git a/docs/02-external-scheduler-api/05-pod-gang-scheduling.md b/docs/02-external-scheduler-api/05-pod-gang-scheduling.md index 1b43a657c..92cee5a8f 100644 --- a/docs/02-external-scheduler-api/05-pod-gang-scheduling.md +++ b/docs/02-external-scheduler-api/05-pod-gang-scheduling.md @@ -56,7 +56,7 @@ So a default pod pipeline filters nodes down to the feasible set, then binpacks most-utilized node that still fits. The pod scheduler reuses the exact `lib` filter/weigher engine of -[The scheduling engine](01-the-scheduling-engine.md), producing the same `Decision`/`History` records as +[The scheduling engine](01-the-scheduling-engine.md), producing the same `Decision` records as every other domain — only the trigger (a watch on pods, not an HTTP call) and the final action (a `Binding`, not a returned ordering) differ. IronCore machine scheduling, next, is the other watch-driven domain, and it commits its choice by setting a field on a Machine rather than binding a pod. diff --git a/docs/02-external-scheduler-api/06-ironcore-machine-scheduling.md b/docs/02-external-scheduler-api/06-ironcore-machine-scheduling.md index 33aad166a..15c7a8e7a 100644 --- a/docs/02-external-scheduler-api/06-ironcore-machine-scheduling.md +++ b/docs/02-external-scheduler-api/06-ironcore-machine-scheduling.md @@ -58,7 +58,7 @@ is added later by implementing and registering steps IronCore machine scheduling completes the chapter's pattern: five domains, one shared engine, differing only in trigger and commit action. Nova, Manila, and Cinder answer HTTP calls and *advise*; pods and machines watch Kubernetes resources and *commit* (a pod `Binding`, a machine `MachinePoolRef`). All five -produce `Decision` and `History` records through the same `lib` pipeline. With the scheduler API covered, +produce `Decision` records through the same `lib` pipeline. With the scheduler API covered, the next chapter turns to the capacity that a pipeline must respect: reservations and inventory. ## Next diff --git a/docs/06-cortex-library/01-multicluster-client.md b/docs/06-cortex-library/01-multicluster-client.md index 661d3ba24..9765efe53 100644 --- a/docs/06-cortex-library/01-multicluster-client.md +++ b/docs/06-cortex-library/01-multicluster-client.md @@ -123,8 +123,8 @@ Setting up multicluster comes down to establishing that trust and declaring the gvks: - kvm.cloud.sap/v1/Hypervisor - kvm.cloud.sap/v1/HypervisorList - - cortex.cloud/v1alpha1/History - - cortex.cloud/v1alpha1/HistoryList + - cobaltcore.cloud/v1alpha1/VirtualMachine + - cobaltcore.cloud/v1alpha1/VirtualMachineList labels: availabilityZone: az-a caCert: | @@ -147,7 +147,7 @@ The rest of this page is a hands-on companion to the concepts above: it stands u multicluster environment on your laptop — a **home** cluster running the controllers and two **remote** clusters (`az-a` and `az-b`) that own their own resources — using three local [kind](https://kind.sigs.k8s.io/) clusters. You then send a scheduling request through Cortex and watch -the resulting `History` land on the remote cluster that owns it, seeing the home/remote routing described +the resulting `VirtualMachine` land on the remote cluster that owns it, seeing the home/remote routing described above work end to end. This is a learning-oriented walkthrough on throwaway clusters. Every command is spelled out inline, along @@ -339,13 +339,16 @@ kubectl config use-context kind-cortex-remote-az-b helm install helm/bundles/cortex-crds --generate-name ``` -Install the external `Hypervisor` CRD on all three clusters: +Install the external `Hypervisor` CRD on all three clusters, plus the vm-operator `VirtualMachine` CRD +(the scheduling result is written as a `VirtualMachine`, routed to the remote that owns the target AZ): ```bash curl -L https://raw.githubusercontent.com/cobaltcore-dev/openstack-hypervisor-operator/refs/heads/main/charts/openstack-hypervisor-operator/crds/kvm.cloud.sap_hypervisors.yaml > /tmp/hypervisor-crd.yaml -kubectl --context kind-cortex-home apply -f /tmp/hypervisor-crd.yaml -kubectl --context kind-cortex-remote-az-a apply -f /tmp/hypervisor-crd.yaml -kubectl --context kind-cortex-remote-az-b apply -f /tmp/hypervisor-crd.yaml +curl -L https://raw.githubusercontent.com/cobaltcore-dev/vm-operator/refs/heads/main/dist/files/crds/cobaltcore.cloud_virtualmachines.yaml > /tmp/virtualmachine-crd.yaml +for ctx in kind-cortex-home kind-cortex-remote-az-a kind-cortex-remote-az-b; do + kubectl --context "$ctx" apply -f /tmp/hypervisor-crd.yaml + kubectl --context "$ctx" apply -f /tmp/virtualmachine-crd.yaml +done ``` #### Configure routing and start Cortex @@ -376,8 +379,8 @@ global: gvks: - kvm.cloud.sap/v1/Hypervisor - kvm.cloud.sap/v1/HypervisorList - - cortex.cloud/v1alpha1/History - - cortex.cloud/v1alpha1/HistoryList + - cobaltcore.cloud/v1alpha1/VirtualMachine + - cobaltcore.cloud/v1alpha1/VirtualMachineList labels: availabilityZone: cortex-remote-az-a caCert: | @@ -386,8 +389,8 @@ $(cat /tmp/root-ca-remote-az-a.pem | sed 's/^/ /') gvks: - kvm.cloud.sap/v1/Hypervisor - kvm.cloud.sap/v1/HypervisorList - - cortex.cloud/v1alpha1/History - - cortex.cloud/v1alpha1/HistoryList + - cobaltcore.cloud/v1alpha1/VirtualMachine + - cobaltcore.cloud/v1alpha1/VirtualMachineList labels: availabilityZone: cortex-remote-az-b caCert: | @@ -410,6 +413,7 @@ metadata: name: hypervisor-1-az-a labels: topology.kubernetes.io/zone: cortex-remote-az-a + topology.kubernetes.io/region: cortex-remote --- apiVersion: kvm.cloud.sap/v1 kind: Hypervisor @@ -417,6 +421,7 @@ metadata: name: hypervisor-2-az-a labels: topology.kubernetes.io/zone: cortex-remote-az-a + topology.kubernetes.io/region: cortex-remote EOF kubectl --context kind-cortex-remote-az-b apply -f - <<'EOF' @@ -426,6 +431,7 @@ metadata: name: hypervisor-1-az-b labels: topology.kubernetes.io/zone: cortex-remote-az-b + topology.kubernetes.io/region: cortex-remote --- apiVersion: kvm.cloud.sap/v1 kind: Hypervisor @@ -433,6 +439,7 @@ metadata: name: hypervisor-2-az-b labels: topology.kubernetes.io/zone: cortex-remote-az-b + topology.kubernetes.io/region: cortex-remote EOF ``` @@ -463,7 +470,7 @@ With Tilt still up, work through the following in another terminal. #### Apply the test pipeline -This is a minimal `filter-weigher` pipeline with `createHistory: true` and a single `filter_correct_az` +This is a minimal `filter-weigher` pipeline with a single `filter_correct_az` step (no weighers, so surviving hosts keep their input order): ```bash @@ -476,7 +483,6 @@ spec: schedulingDomain: nova description: Minimal test pipeline for the multicluster guide. type: filter-weigher - createHistory: true filters: - name: filter_correct_az weighers: [] @@ -546,7 +552,7 @@ curl -s -X POST http://localhost:8001/scheduler/nova/external \ "vcpu_weight": 0, "disabled": false, "is_public": true, - "extra_specs": {} + "extra_specs": {"capabilities:hypervisor_type": "qemu"} } }, "request_level_params": { @@ -600,31 +606,40 @@ curl -s -X POST http://localhost:8001/scheduler/nova/external \ A `200` response with a JSON body listing only `az-b` hosts confirms the scheduler filtered correctly. -#### Verify the History landed on the correct remote +#### Verify the VirtualMachine landed on the correct remote + +Because the request targeted `cortex-remote-az-b`, Cortex writes the scheduling result as a +`VirtualMachine` and the multicluster client should have routed it to that remote — not to the home +cluster or the other remote. + +> [!NOTE] +> VM creation is gated by the nova scheduler's `VirtualMachineCreation` feature gate. The guide's +> default overrides enable it; if you see no `VirtualMachine` below, confirm the gate is on and that the +> request's flavor carries `capabilities:hypervisor_type: qemu` (set above) so Cortex treats it as a KVM +> placement. -Because the pipeline has `createHistory: true` and the request targeted `cortex-remote-az-b`, the -multicluster client should have written the `History` resource to that remote — not to the home cluster -or the other remote. Check each cluster: +Check each cluster: ```bash echo "--- kind-cortex-home ---" -kubectl --context kind-cortex-home get histories 2>/dev/null || echo " (none)" +kubectl --context kind-cortex-home get virtualmachines 2>/dev/null || echo " (none)" echo "--- kind-cortex-remote-az-a ---" -kubectl --context kind-cortex-remote-az-a get histories 2>/dev/null || echo " (none)" +kubectl --context kind-cortex-remote-az-a get virtualmachines 2>/dev/null || echo " (none)" echo "--- kind-cortex-remote-az-b ---" -kubectl --context kind-cortex-remote-az-b get histories 2>/dev/null || echo " (none)" +kubectl --context kind-cortex-remote-az-b get virtualmachines 2>/dev/null || echo " (none)" ``` -You should see the `History` named `nova-cortex-test-instance-001` only in `kind-cortex-remote-az-b`. -Describe it to see the full scheduling decision — which pipeline ran, which host was selected, and the -explanation for each filter step: +You should see the `VirtualMachine` named `cortex-test-instance-001` only in `kind-cortex-remote-az-b`. +Describe it to see the full scheduling result — the target host under `spec.targetHost`, the ranked +`spec.cortex.hostCandidates`, and `spec.cortex.explanation`, the human-readable summary of why that host +won: ```bash -kubectl --context kind-cortex-remote-az-b describe history nova-cortex-test-instance-001 +kubectl --context kind-cortex-remote-az-b describe virtualmachine cortex-test-instance-001 ``` This confirms end-to-end multicluster routing: the scheduling request entered through the home cluster, -the `filter_correct_az` filter ran against hosts from both remotes, and the resulting `History` was +the `filter_correct_az` filter ran against hosts from both remotes, and the resulting `VirtualMachine` was written to the remote whose labels matched the request's availability zone. ### Step 3 — Clean up diff --git a/helm/bundles/cortex-cinder/values.yaml b/helm/bundles/cortex-cinder/values.yaml index ef471d343..a4873c3ef 100644 --- a/helm/bundles/cortex-cinder/values.yaml +++ b/helm/bundles/cortex-cinder/values.yaml @@ -92,8 +92,6 @@ cortex: &cortex gvks: - cortex.cloud/v1alpha1/Decision - cortex.cloud/v1alpha1/DecisionList - - cortex.cloud/v1alpha1/History - - cortex.cloud/v1alpha1/HistoryList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline @@ -134,8 +132,6 @@ cortex-scheduling-controllers: component: cinder-scheduling enabledControllers: - cinder-decisions-pipeline-controller - enabledTasks: - - cinder-history-cleanup-task cortex-knowledge-controllers: <<: *cortex diff --git a/helm/bundles/cortex-ironcore/values.yaml b/helm/bundles/cortex-ironcore/values.yaml index 5017ef0bf..6f577b04b 100644 --- a/helm/bundles/cortex-ironcore/values.yaml +++ b/helm/bundles/cortex-ironcore/values.yaml @@ -35,8 +35,6 @@ cortex: gvks: - cortex.cloud/v1alpha1/Decision - cortex.cloud/v1alpha1/DecisionList - - cortex.cloud/v1alpha1/History - - cortex.cloud/v1alpha1/HistoryList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline diff --git a/helm/bundles/cortex-manila/values.yaml b/helm/bundles/cortex-manila/values.yaml index 993fcc9ab..34a8c9702 100644 --- a/helm/bundles/cortex-manila/values.yaml +++ b/helm/bundles/cortex-manila/values.yaml @@ -92,8 +92,6 @@ cortex: &cortex gvks: - cortex.cloud/v1alpha1/Decision - cortex.cloud/v1alpha1/DecisionList - - cortex.cloud/v1alpha1/History - - cortex.cloud/v1alpha1/HistoryList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline @@ -134,8 +132,6 @@ cortex-scheduling-controllers: component: manila-scheduling enabledControllers: - manila-decisions-pipeline-controller - enabledTasks: - - manila-history-cleanup-task cortex-knowledge-controllers: <<: *cortex diff --git a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml index 734063c54..7fc4f0223 100644 --- a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml @@ -50,7 +50,6 @@ spec: impl: multicluster_object_count_kpi opts: gvks: - - cortex.cloud/v1alpha1/HistoryList - cortex.cloud/v1alpha1/ReservationList - cortex.cloud/v1alpha1/CommittedResourceList - kvm.cloud.sap/v1/HypervisorList diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index 5fa3230e0..a8278993f 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -646,341 +646,4 @@ spec: Ensures hosts meet the compute capabilities required by the flavor extra specs (e.g., architecture, maxphysaddr bits). weighers: [] ---- -apiVersion: cortex.cloud/v1alpha1 -kind: Pipeline -metadata: - name: kvm-general-purpose-load-balancing-no-history -spec: - schedulingDomain: nova - description: | - Variant of kvm-general-purpose-load-balancing used for committed-resource - reservation scheduling and capacity probes. Identical filter/weigher chain - but does not write placement history, to avoid polluting history with - internal CR controller calls. - type: filter-weigher - ignorePreselection: true - createHistory: false - filters: - - name: filter_status_conditions - description: | - This step will filter out hosts for which the hypervisor status conditions - do not meet the expected values, for example, that the hypervisor is ready - and not disabled. - - name: filter_correct_az - description: | - This step will filter out hosts whose aggregate information indicates they - are not placed in the requested availability zone. - - name: filter_host_instructions - description: | - This step will consider the `ignore_hosts` and `force_hosts` instructions - from the nova scheduler request spec to filter out or exclusively allow - certain hosts. - - name: filter_image_properties - description: | - This step will filter out hosts for which the hypervisor type does not match - the one specified in the image properties, for example, filtering out all - known KVM hypervisors if the image requires a different hypervisor type. - - name: filter_capabilities - description: | - This step will filter out hosts that do not meet the compute capabilities - requested by the nova flavor extra specs, like `{"arch": "x86_64", - "maxphysaddr:bits": 46, ...}`. - - Note: currently, advanced boolean/numeric operators for the capabilities - like `>`, `!`, ... are not supported because they are not used by any of our - flavors in production. - - name: filter_has_requested_traits - description: | - This step filters hosts that do not have the requested traits given by the - nova flavor extra spec: "trait:": "forbidden" means the host must - not have the specified trait. "trait:": "required" means the host - must have the specified trait. - - name: filter_external_customer - description: | - This step prefix-matches the domain name for external customer domains and - filters out hosts that are not intended for external customers. It considers - the `CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE` trait on hosts as well as the - `domain_name` scheduler hint from the nova request spec. - params: - - {key: domainNamePrefixes, stringListValue: {{ .Values.kvm.filterExternalCustomer.domainNamePrefixes | toJson }}} - {{- with .Values.kvm.filterExternalCustomer.ignoredDomainNames }} - - {key: ignoredDomainNames, stringListValue: {{ . | toJson }}} - {{- end }} - - name: filter_has_accelerators - description: | - This step will filter out hosts without the trait `COMPUTE_ACCELERATORS` if - the nova flavor extra specs request accelerators via "accel:device_profile". - - name: filter_instance_group_affinity - description: | - This step selects hosts in the instance group specified in the nova - scheduler request spec. - - name: filter_instance_group_anti_affinity - description: | - This step selects hosts not in the instance group specified in the nova - scheduler request spec, but only until the max_server_per_host limit is - reached (default = 1). - - name: filter_has_enough_capacity - description: | - This step will filter out hosts that do not have enough available capacity - to host the requested flavor. If enabled, this step will subtract the - current reservations residing on this host from the available capacity. - params: - - {key: lockReserved, boolValue: false} - - name: filter_allowed_projects - description: | - This step filters hosts based on allowed projects defined in the - hypervisor resource. Note that hosts allowing all projects are still - accessible and will not be filtered out. In this way some hypervisors - are made accessible to some projects only. - - name: filter_aggregate_metadata - description: | - This step filters hosts based on metadata defined in their aggregates. For - example, if an aggregate has the metadata "filter_tenant_id": "", - only hosts in that aggregate that match the project ID in the nova request - will pass this filter. - - name: filter_live_migratable - description: | - This step ensures that the target host of a live migration can accept - the migrating VM, by checking cpu architecture, cpu features, emulated - devices, and cpu modes. - - name: filter_requested_destination - description: | - This step filters hosts based on the `requested_destination` instruction - from the nova scheduler request spec. It supports filtering by host and - by aggregates. Aggregates use AND logic between list elements, with - comma-separated UUIDs within an element using OR logic. - - name: filter_quota_enforcement - description: | - This step enforces project quota by checking whether the request has - headroom under the project's committed resources or pay-as-you-go quota. - If a matching CommittedResource has unused capacity, the request is accepted. - Otherwise, PAYG headroom is checked for ram, cores, and instances. - Rejects all hosts if neither tier has headroom. - - Runs in shadow mode by default: the filter logs and emits the - cortex_nova_filter_quota_enforcement_decisions_total metric for - would-be rejects but never actually removes hosts. To globally reject - requests without headroom, add `params: [{key: enforce, boolValue: true}]` - to this step. The shadow default is intentional so that newly-rolled-out - releases never silently start rejecting requests. - weighers: - - name: kvm_prefer_smaller_hosts - params: - - {key: resourceWeights, floatMapValue: {"memory": 1.0}} - description: | - This step pulls virtual machines onto smaller hosts (by capacity). This - ensures that larger hosts are not overly fragmented with small VMs, - and can still accommodate larger VMs when they need to be scheduled. - - name: kvm_instance_group_soft_affinity - description: | - This weigher implements the "soft affinity" and "soft anti-affinity" policy - for instance groups in nova. - - It assigns a weight to each host based on how many instances of the same - instance group are already running on that host. The more instances of the - same group on a host, the lower (for soft-anti-affinity) or higher - (for soft-affinity) the weight, which makes it less likely or more likely, - respectively, for the scheduler to choose that host for new instances of the - same group. - - name: kvm_binpack - multiplier: -1.0 # inverted = balancing - params: - - {key: resourceWeights, floatMapValue: {"memory": 1.0}} - description: | - This step implements a balancing weigher for workloads on kvm hypervisors, - which is the opposite of binpacking. Instead of pulling the requested vm - into the smallest gaps possible, it spreads the load to ensure - workloads are balanced across hosts. In this pipeline, the balancing will - focus on general purpose virtual machines. - - name: kvm_failover_evacuation - description: | - This weigher prefers hosts with active failover reservations during - evacuation requests. Hosts matching a failover reservation where the - VM is allocated get a higher weight, encouraging placement on - pre-reserved failover capacity. For non-evacuation requests, this - weigher has no effect. - - name: kvm_committed_resource_reservation - description: | - This weigher boosts hosts that have a ready CommittedResourceReservation - matching the request's project, resource group, and availability zone, - with enough free memory capacity for the requested VM. Hosts without a - matching reservation or without enough free capacity receive a lower weight. - - name: kvm_cr_migration_slot - description: | - During live migrations of VMs that occupy a committed-resource reservation - slot, this weigher boosts hosts that either have a ready matching CR slot - with enough remaining capacity for the full slot size, or have enough free - memory for the slot to follow via the reconciler. Hosts that can accommodate - neither receive a lower weight. If the VM has no confirmed CR slot, all - candidates receive equal weight. - Only activates for live_migrate requests. All other intents pass through. - Emits cortex_nova_weigh_cr_migration_slot_requests_total. ---- -apiVersion: cortex.cloud/v1alpha1 -kind: Pipeline -metadata: - name: kvm-hana-bin-packing-no-history -spec: - schedulingDomain: nova - description: | - Variant of kvm-hana-bin-packing used for committed-resource reservation - scheduling and capacity probes. Identical filter/weigher chain but does not - write placement history, to avoid polluting history with internal CR - controller calls. - type: filter-weigher - ignorePreselection: true - createHistory: false - filters: - - name: filter_status_conditions - description: | - This step will filter out hosts for which the hypervisor status conditions - do not meet the expected values, for example, that the hypervisor is ready - and not disabled. - - name: filter_correct_az - description: | - This step will filter out hosts whose aggregate information indicates they - are not placed in the requested availability zone. - - name: filter_host_instructions - description: | - This step will consider the `ignore_hosts` and `force_hosts` instructions - from the nova scheduler request spec to filter out or exclusively allow - certain hosts. - - name: filter_image_properties - description: | - This step will filter out hosts for which the hypervisor type does not match - the one specified in the image properties, for example, filtering out all - known KVM hypervisors if the image requires a different hypervisor type. - - name: filter_capabilities - description: | - This step will filter out hosts that do not meet the compute capabilities - requested by the nova flavor extra specs, like `{"arch": "x86_64", - "maxphysaddr:bits": 46, ...}`. - - Note: currently, advanced boolean/numeric operators for the capabilities - like `>`, `!`, ... are not supported because they are not used by any of our - flavors in production. - - name: filter_has_requested_traits - description: | - This step filters hosts that do not have the requested traits given by the - nova flavor extra spec: "trait:": "forbidden" means the host must - not have the specified trait. "trait:": "required" means the host - must have the specified trait. - - name: filter_external_customer - description: | - This step prefix-matches the domain name for external customer domains and - filters out hosts that are not intended for external customers. It considers - the `CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE` trait on hosts as well as the - `domain_name` scheduler hint from the nova request spec. - params: - - {key: domainNamePrefixes, stringListValue: {{ .Values.kvm.filterExternalCustomer.domainNamePrefixes | toJson }}} - {{- with .Values.kvm.filterExternalCustomer.ignoredDomainNames }} - - {key: ignoredDomainNames, stringListValue: {{ . | toJson }}} - {{- end }} - - name: filter_has_accelerators - description: | - This step will filter out hosts without the trait `COMPUTE_ACCELERATORS` if - the nova flavor extra specs request accelerators via "accel:device_profile". - - name: filter_instance_group_affinity - description: | - This step selects hosts in the instance group specified in the nova - scheduler request spec. - - name: filter_instance_group_anti_affinity - description: | - This step selects hosts not in the instance group specified in the nova - scheduler request spec, but only until the max_server_per_host limit is - reached (default = 1). - - name: filter_has_enough_capacity - description: | - This step will filter out hosts that do not have enough available capacity - to host the requested flavor. If enabled, this step will subtract the - current reservations residing on this host from the available capacity. - params: - - {key: lockReserved, boolValue: false} - - name: filter_allowed_projects - description: | - This step filters hosts based on allowed projects defined in the - hypervisor resource. Note that hosts allowing all projects are still - accessible and will not be filtered out. In this way some hypervisors - are made accessible to some projects only. - - name: filter_aggregate_metadata - description: | - This step filters hosts based on metadata defined in their aggregates. For - example, if an aggregate has the metadata "filter_tenant_id": "", - only hosts in that aggregate that match the project ID in the nova request - will pass this filter. - - name: filter_live_migratable - description: | - This step ensures that the target host of a live migration can accept - the migrating VM, by checking cpu architecture, cpu features, emulated - devices, and cpu modes. - - name: filter_requested_destination - description: | - This step filters hosts based on the `requested_destination` instruction - from the nova scheduler request spec. It supports filtering by host and - by aggregates. Aggregates use AND logic between list elements, with - comma-separated UUIDs within an element using OR logic. - - name: filter_quota_enforcement - description: | - This step enforces project quota by checking whether the request has - headroom under the project's committed resources or pay-as-you-go quota. - If a matching CommittedResource has unused capacity, the request is accepted. - Otherwise, PAYG headroom is checked for ram, cores, and instances. - Rejects all hosts if neither tier has headroom. - - Runs in shadow mode by default: the filter logs and emits the - cortex_nova_filter_quota_enforcement_decisions_total metric for - would-be rejects but never actually removes hosts. To globally reject - requests without headroom, add `params: [{key: enforce, boolValue: true}]` - to this step. The shadow default is intentional so that newly-rolled-out - releases never silently start rejecting requests. - weighers: - - name: kvm_prefer_smaller_hosts - params: - - {key: resourceWeights, floatMapValue: {"memory": 1.0}} - description: | - This step pulls virtual machines onto smaller hosts (by capacity). This - ensures that larger hosts are not overly fragmented with small VMs, - and can still accommodate larger VMs when they need to be scheduled. - - name: kvm_instance_group_soft_affinity - description: | - This weigher implements the "soft affinity" and "soft anti-affinity" policy - for instance groups in nova. - It assigns a weight to each host based on how many instances of the same - instance group are already running on that host. The more instances of the - same group on a host, the lower (for soft-anti-affinity) or higher - (for soft-affinity) the weight, which makes it less likely or more likely, - respectively, for the scheduler to choose that host for new instances of the - same group. - - name: kvm_binpack - params: - - {key: resourceWeights, floatMapValue: {"memory": 1.0}} - description: | - This step implements a binpacking weigher for workloads on kvm hypervisors. - It pulls the requested vm into the smallest gaps possible, to ensure - other hosts with less allocation stay free for bigger vms. - In this pipeline, the binpacking will focus on hana virtual machines. - - name: kvm_failover_evacuation - description: | - This weigher prefers hosts with active failover reservations during - evacuation requests. Hosts matching a failover reservation where the - VM is allocated get a higher weight, encouraging placement on - pre-reserved failover capacity. For non-evacuation requests, this - weigher has no effect. - - name: kvm_committed_resource_reservation - description: | - This weigher boosts hosts that have a ready CommittedResourceReservation - matching the request's project, resource group, and availability zone, - with enough free memory capacity for the requested VM. Hosts without a - matching reservation or without enough free capacity receive a lower weight. - - name: kvm_cr_migration_slot - description: | - During live migrations of VMs that occupy a committed-resource reservation - slot, this weigher boosts hosts that either have a ready matching CR slot - with enough remaining capacity for the full slot size, or have enough free - memory for the slot to follow via the reconciler. Hosts that can accommodate - neither receive a lower weight. If the VM has no confirmed CR slot, all - candidates receive equal weight. - Only activates for live_migrate requests. All other intents pass through. - Emits cortex_nova_weigh_cr_migration_slot_requests_total. {{- end }} diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 6db8c0322..b47acd8e6 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -123,8 +123,6 @@ cortex: &cortex gvks: - cortex.cloud/v1alpha1/Decision - cortex.cloud/v1alpha1/DecisionList - - cortex.cloud/v1alpha1/History - - cortex.cloud/v1alpha1/HistoryList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline @@ -190,7 +188,6 @@ cortex-scheduling-controllers: - quota-controller - capacity-controller enabledTasks: - - nova-history-cleanup-task - commitments-sync-task # required for committed resources # Feature gates control optional capabilities within running components. # Enables VM allocation write-back into Reservation slots and no-host-found diff --git a/helm/bundles/cortex-pods/values.yaml b/helm/bundles/cortex-pods/values.yaml index fb4efaca1..efd5ee9b1 100644 --- a/helm/bundles/cortex-pods/values.yaml +++ b/helm/bundles/cortex-pods/values.yaml @@ -35,8 +35,6 @@ cortex: gvks: - cortex.cloud/v1alpha1/Decision - cortex.cloud/v1alpha1/DecisionList - - cortex.cloud/v1alpha1/History - - cortex.cloud/v1alpha1/HistoryList - cortex.cloud/v1alpha1/Descheduling - cortex.cloud/v1alpha1/DeschedulingList - cortex.cloud/v1alpha1/Pipeline diff --git a/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml b/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml index a9fe3c78e..ec4c99649 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_decisions.yaml @@ -42,7 +42,8 @@ spec: description: |- Currently the Decision CRD is an in-memory scheduling object used by the external scheduler API and filter-weigher pipelines to compute a placement result. It is currently NOT persisted - to etcd — the scheduling outcome is recorded in the History CRD instead. + to etcd — the scheduling outcome (including a human-readable explanation) is written onto the + scheduled resource instead, e.g. the nova VM's spec.cortex fields. The long-term shape of this CRD is still under discussion; until that is settled the Decision serves only as a transient carrier within a single scheduling run. diff --git a/helm/library/cortex/files/crds/cortex.cloud_histories.yaml b/helm/library/cortex/files/crds/cortex.cloud_histories.yaml deleted file mode 100644 index 5d72687b7..000000000 --- a/helm/library/cortex/files/crds/cortex.cloud_histories.yaml +++ /dev/null @@ -1,303 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.23.0 - name: histories.cortex.cloud -spec: - group: cortex.cloud - names: - kind: History - listKind: HistoryList - plural: histories - singular: history - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .spec.schedulingDomain - name: Domain - type: string - - jsonPath: .spec.resourceID - name: Resource ID - type: string - - jsonPath: .spec.availabilityZone - name: AZ - type: string - - jsonPath: .status.current.targetHost - name: Target Host - type: string - - jsonPath: .status.conditions[?(@.type=='Ready')].reason - name: Status - type: string - - jsonPath: .status.current.timestamp - name: Last Scheduled - type: date - - jsonPath: .metadata.creationTimestamp - name: Created - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - The history is a CRD that provides a record of past scheduling decisions for a given resource (e.g., a nova instance). - A new history entry is created for each scheduling decision, and the most recent decision is stored in the status.current field. The history is capped at 10 entries to prevent unbounded growth. - This CRD is designed to be used by an operations team to troubleshoot scheduling decisions and understand the context around why a particular host was selected (or not selected) for a resource. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: Spec defines the desired state of History. - properties: - availabilityZone: - description: |- - The availability zone of the resource, if known. Only set for scheduling - domains that provide AZ information (e.g., Nova). - type: string - resourceID: - description: The resource ID this history belongs to (e.g., the UUID - of a nova instance). - type: string - schedulingDomain: - description: The scheduling domain this object with the history belongs - to. - type: string - required: - - resourceID - - schedulingDomain - type: object - status: - description: Status defines the observed state of History. - properties: - conditions: - description: Conditions represent the latest available observations - of the history's state. - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - current: - description: Current represents the latest scheduling decision with - full context. - properties: - explanation: - description: A human-readable explanation of the scheduling decision. - type: string - intent: - description: The intent of the decision (e.g., initial scheduling, - rescheduling, etc.). - type: string - orderedHosts: - description: The top hosts ordered by score (limited to 3). - items: - type: string - maxItems: 3 - type: array - pipelineRef: - description: The pipeline that was used for the decision. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - successful: - description: Whether the scheduling decision was successful. - type: boolean - targetHost: - description: The target host selected for the resource. nil when - no host was found. - type: string - timestamp: - description: The timestamp of when the decision was made. - format: date-time - type: string - required: - - intent - - pipelineRef - - successful - - timestamp - type: object - history: - description: History of past scheduling decisions (limited to last - 10). - items: - properties: - intent: - description: The intent of the decision (e.g., initial scheduling, - rescheduling, etc.). - type: string - orderedHosts: - description: |- - The top hosts ordered by score for the decision (limited to 3). - This is not a complete list of all candidates — only the highest-ranked - hosts are retained to keep the history compact. - items: - type: string - maxItems: 3 - type: array - pipelineRef: - description: The pipeline that was used for the decision. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: |- - If referring to a piece of an object instead of an entire object, this string - should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within a pod, this would take on a value like: - "spec.containers{name}" (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to have some well-defined way of - referencing a part of an object. - type: string - kind: - description: |- - Kind of the referent. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - name: - description: |- - Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - type: string - namespace: - description: |- - Namespace of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ - type: string - resourceVersion: - description: |- - Specific resourceVersion to which this reference is made, if any. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency - type: string - uid: - description: |- - UID of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids - type: string - type: object - x-kubernetes-map-type: atomic - successful: - description: Whether the scheduling decision was successful. - type: boolean - timestamp: - description: The timestamp of when the decision was made. - format: date-time - type: string - required: - - intent - - pipelineRef - - timestamp - type: object - type: array - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} diff --git a/internal/scheduling/cinder/filter_weigher_pipeline_controller.go b/internal/scheduling/cinder/filter_weigher_pipeline_controller.go index bb1ea8e31..6d2593147 100644 --- a/internal/scheduling/cinder/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/cinder/filter_weigher_pipeline_controller.go @@ -113,17 +113,13 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision } result, err := pipeline.Run(request) - // Persist the result on the decision before upserting history so that - // CreateOrUpdateHistory can observe the target host and ordered hosts. - // On error the result is empty/meaningless, so only set it on success. if err == nil { decision.Status.Result = &result } - if !request.Options.SkipHistory { - if upsertErr := c.HistoryManager.CreateOrUpdateHistory(ctx, decision, nil, err); upsertErr != nil { - log.Error(upsertErr, "failed to create/update history") - } - } + // Record a human-readable explanation of the decision for downstream + // consumers. On error the result is nil and the explanation carries the + // failure reason. + decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) if err != nil { log.Error(err, "failed to run pipeline") return err @@ -149,7 +145,6 @@ func (c *FilterWeigherPipelineController) InitPipeline( func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, mcl *multicluster.Client) error { c.Initializer = c c.SchedulingDomain = v1alpha1.SchedulingDomainCinder - c.HistoryManager = lib.HistoryClient{Client: mcl, Recorder: mcl.GetEventRecorder("cortex-cinder-scheduler")} if err := mgr.Add(manager.RunnableFunc(c.InitAllPipelines)); err != nil { return err } diff --git a/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go b/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go index 05d474b60..8ec56c40f 100644 --- a/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/cinder/filter_weigher_pipeline_controller_test.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "testing" - "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -47,7 +46,7 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { }, Weights: map[string]float64{"cinder-volume-1": 1.0, "cinder-volume-2": 0.5}, Pipeline: "test-pipeline", - Options: scheduling.Options{SkipHistory: true}, + Options: scheduling.Options{}, } cinderRaw, err := json.Marshal(cinderRequest) @@ -250,13 +249,11 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) } tests := []struct { - name string - decision *v1alpha1.Decision - pipelineConfig *v1alpha1.Pipeline - createHistory bool - expectError bool - expectHistoryCreated bool - expectResult bool + name string + decision *v1alpha1.Decision + pipelineConfig *v1alpha1.Pipeline + expectError bool + expectResult bool }{ { name: "successful decision processing with creation", @@ -287,10 +284,8 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) Weighers: []v1alpha1.WeigherSpec{}, }, }, - createHistory: true, - expectError: false, - expectHistoryCreated: true, - expectResult: true, + expectError: false, + expectResult: true, }, { name: "successful decision processing without creation", @@ -320,10 +315,8 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) Weighers: []v1alpha1.WeigherSpec{}, }, }, - createHistory: false, - expectError: false, - expectHistoryCreated: false, - expectResult: true, + expectError: false, + expectResult: true, }, { name: "pipeline not configured", @@ -342,10 +335,9 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) }, }, }, - pipelineConfig: nil, - expectError: true, - expectHistoryCreated: false, - expectResult: false, + pipelineConfig: nil, + expectError: true, + expectResult: false, }, { name: "decision without cinderRaw spec", @@ -373,10 +365,8 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) Weighers: []v1alpha1.WeigherSpec{}, }, }, - createHistory: true, - expectError: true, - expectHistoryCreated: false, - expectResult: false, + expectError: true, + expectResult: false, }, } @@ -390,7 +380,7 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) client := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}, &v1alpha1.History{}). + WithStatusSubresource(&v1alpha1.Decision{}). Build() controller := &FilterWeigherPipelineController{ @@ -398,7 +388,6 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) Client: client, Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), - HistoryManager: lib.HistoryClient{Client: client}, }, Monitor: lib.FilterWeigherPipelineMonitor{}, } @@ -414,7 +403,7 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) if tt.decision.Spec.CinderRaw != nil { req := cinderRequest - req.Options = scheduling.Options{SkipHistory: !tt.createHistory} + req.Options = scheduling.Options{} raw, marshalErr := json.Marshal(req) if marshalErr != nil { t.Fatalf("Failed to marshal request with options: %v", marshalErr) @@ -431,23 +420,7 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) t.Errorf("Expected no error but got: %v", err) } - // Check if history CRD was created when expected - if tt.expectHistoryCreated { - var histories v1alpha1.HistoryList - deadline := time.Now().Add(2 * time.Second) - for { - if err := client.List(context.Background(), &histories); err != nil { - t.Fatalf("Failed to list histories: %v", err) - } - if len(histories.Items) > 0 { - break - } - if time.Now().After(deadline) { - t.Fatal("timed out waiting for history CRD to be created") - } - time.Sleep(5 * time.Millisecond) - } - } else if !tt.expectError { + if !tt.expectError { if tt.expectResult && tt.decision.Status.Result == nil { t.Error("expected decision result to be set in original decision object") } diff --git a/internal/scheduling/cinder/history_cleanup.go b/internal/scheduling/cinder/history_cleanup.go deleted file mode 100644 index 647c3426a..000000000 --- a/internal/scheduling/cinder/history_cleanup.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package cinder - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - "net/http" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - - "github.com/cobaltcore-dev/cortex/pkg/keystone" - "github.com/cobaltcore-dev/cortex/pkg/sso" - "github.com/gophercloud/gophercloud/v2" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type HistoryCleanupConfig struct { - // Secret ref to keystone credentials stored in a k8s secret. - KeystoneSecretRef corev1.SecretReference `json:"keystoneSecretRef"` - // Secret ref to SSO credentials stored in a k8s secret, if applicable. - SSOSecretRef *corev1.SecretReference `json:"ssoSecretRef"` -} - -// Delete all history entries for cinder volumes that have been deleted. -func HistoryCleanup(ctx context.Context, client client.Client, conf HistoryCleanupConfig) error { - var authenticatedHTTP = http.DefaultClient - if conf.SSOSecretRef != nil { - var err error - authenticatedHTTP, err = sso.Connector{Client: client}. - FromSecretRef(ctx, *conf.SSOSecretRef) - if err != nil { - return err - } - } - authenticatedKeystone, err := keystone.Connector{Client: client, HTTPClient: authenticatedHTTP}. - FromSecretRef(ctx, conf.KeystoneSecretRef) - if err != nil { - return err - } - pc := authenticatedKeystone.Client() - cinderURL, err := pc.EndpointLocator(gophercloud.EndpointOpts{ - Type: "volumev3", - Availability: gophercloud.Availability(authenticatedKeystone.Availability()), - }) - if err != nil { - return err - } - cinderSC := &gophercloud.ServiceClient{ - ProviderClient: pc, - Endpoint: cinderURL, - Microversion: "3.70", - } - - initialURL := cinderSC.Endpoint + "volumes/detail?all_tenants=true" - var nextURL = &initialURL - var volumes []struct { - ID string `json:"id"` - } - - for nextURL != nil { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, *nextURL, http.NoBody) - if err != nil { - return err - } - req.Header.Set("X-Auth-Token", cinderSC.Token()) - req.Header.Set("OpenStack-API-Version", "volume "+cinderSC.Microversion) - resp, err := cinderSC.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - var list struct { - Volumes []struct { - ID string `json:"id"` - } `json:"volumes"` - Links []struct { - Rel string `json:"rel"` - Href string `json:"href"` - } `json:"volumes_links"` - } - err = json.NewDecoder(resp.Body).Decode(&list) - if err != nil { - return err - } - volumes = append(volumes, list.Volumes...) - nextURL = nil - for _, link := range list.Links { - if link.Rel == "next" { - nextURL = &link.Href - break - } - } - } - - slog.Info("found volumes", "count", len(volumes)) - volumesByID := make(map[string]struct{}) - for _, volume := range volumes { - volumesByID[volume.ID] = struct{}{} - } - - // List all history and delete those whose volume no longer exists. - historyList := &v1alpha1.HistoryList{} - if err := client.List(ctx, historyList); err != nil { - return err - } - for _, history := range historyList.Items { - // Skip non-cinder history entries. - if history.Spec.SchedulingDomain != v1alpha1.SchedulingDomainCinder { - continue - } - // Skip history entries for which the volume still exists. - if _, ok := volumesByID[history.Spec.ResourceID]; ok { - continue - } - // Delete the history entry since the volume no longer exists. - slog.Info("deleting history entry for deleted volume", "history", history.Name, "volumeID", history.Spec.ResourceID) - if err := client.Delete(ctx, &history); err != nil { - return err - } - } - return nil -} diff --git a/internal/scheduling/cinder/history_cleanup_test.go b/internal/scheduling/cinder/history_cleanup_test.go deleted file mode 100644 index 5ec40c724..000000000 --- a/internal/scheduling/cinder/history_cleanup_test.go +++ /dev/null @@ -1,388 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package cinder - -import ( - "context" - "encoding/json" - "errors" - "net/http" - "net/http/httptest" - "testing" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" -) - -func TestCleanupCinder(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add corev1 scheme: %v", err) - } - - tests := []struct { - name string - histories []v1alpha1.History - expectError bool - authError bool - endpointError bool - mockServerError bool - emptyVolumesError bool - mockVolumes []mockVolume - expectedDeleted []string - }{ - { - name: "handle authentication error", - histories: []v1alpha1.History{}, - authError: true, - expectError: true, - }, - { - name: "handle endpoint error", - histories: []v1alpha1.History{}, - endpointError: true, - expectError: true, - }, - { - name: "handle server error", - histories: []v1alpha1.History{}, - mockServerError: true, - expectError: true, - }, - { - name: "handle empty volumes case", - histories: []v1alpha1.History{}, - emptyVolumesError: true, - expectError: false, - }, - { - name: "delete history for non-existent volumes", - histories: []v1alpha1.History{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-existing-volume", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - ResourceID: "volume-exists", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-deleted-volume", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - ResourceID: "volume-deleted", - }, - }, - }, - mockVolumes: []mockVolume{ - {ID: "volume-exists"}, - }, - expectedDeleted: []string{"history-deleted-volume"}, - expectError: false, - }, - { - name: "keep history for existing volumes", - histories: []v1alpha1.History{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-volume-1", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - ResourceID: "volume-1", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-volume-2", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - ResourceID: "volume-2", - }, - }, - }, - mockVolumes: []mockVolume{ - {ID: "volume-1"}, - {ID: "volume-2"}, - }, - expectedDeleted: []string{}, - expectError: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := make([]client.Object, len(tt.histories)) - for i := range tt.histories { - objects[i] = &tt.histories[i] - } - - // Create mock Cinder server first - cinderServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if tt.mockServerError { - w.WriteHeader(http.StatusInternalServerError) - return - } - - // Handle volumes list endpoint - if r.URL.Path == "/volumes" || r.URL.Path == "/volumes/detail" { - w.Header().Set("Content-Type", "application/json") - if tt.emptyVolumesError { - // Return empty volumes list - volumesResponse := map[string]any{ - "volumes": []mockVolume{}, - } - err := json.NewEncoder(w).Encode(volumesResponse) - if err != nil { - t.Errorf("Failed to encode volumes response: %v", err) - } - return - } - - volumesResponse := map[string]any{ - "volumes": tt.mockVolumes, - } - err := json.NewEncoder(w).Encode(volumesResponse) - if err != nil { - t.Errorf("Failed to encode volumes response: %v", err) - } - return - } - - // Default response for other endpoints - w.WriteHeader(http.StatusNotFound) - })) - defer cinderServer.Close() - - // Create mock Keystone server - keystoneServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if tt.authError { - w.WriteHeader(http.StatusUnauthorized) - return - } - - w.Header().Set("Content-Type", "application/json") - - // Handle different Keystone API endpoints - switch r.URL.Path { - case "/", "/v3", "/v3/": - // Handle version discovery - return supported versions - versionResponse := map[string]any{ - "versions": map[string]any{ - "values": []map[string]any{ - { - "status": "stable", - "id": "v3.0", - "links": []map[string]any{ - { - "href": "http://" + r.Host + "/v3/", - "rel": "self", - }, - }, - }, - }, - }, - } - err := json.NewEncoder(w).Encode(versionResponse) - if err != nil { - t.Errorf("Failed to encode version response: %v", err) - } - case "/v3/auth/tokens": - // Set the correct status code that gophercloud expects - w.WriteHeader(http.StatusCreated) - - // Mock token response with proper structure for gophercloud - tokenResponse := map[string]any{ - "token": map[string]any{ - "methods": []string{"password"}, - "expires_at": "2030-01-01T00:00:00Z", - "project": map[string]any{ - "domain": map[string]any{ - "id": "default", - "name": "default", - }, - "id": "test-project-id", - "name": "test-project", - }, - "catalog": []map[string]any{ - { - "type": "volumev3", - "id": "cinder-service-id", - "name": "cinderv3", - "endpoints": []map[string]any{ - { - "region_id": "RegionOne", - "url": cinderServer.URL, - "region": "RegionOne", - "interface": "public", - "id": "cinder-endpoint-id", - }, - }, - }, - }, - "user": map[string]any{ - "domain": map[string]any{ - "id": "default", - "name": "default", - }, - "id": "test-user-id", - "name": "test-user", - }, - }, - } - if tt.endpointError { - // Don't include volumev3 service in catalog - tokenResponse["token"].(map[string]any)["catalog"] = []map[string]any{} - } - - // Set the token in the header for gophercloud - w.Header().Set("X-Subject-Token", "mock-token-id") - err := json.NewEncoder(w).Encode(tokenResponse) - if err != nil { - t.Errorf("Failed to encode token response: %v", err) - } - default: - w.WriteHeader(http.StatusNotFound) - } - })) - defer keystoneServer.Close() - - // Add the keystone secret object - objects = append(objects, &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "keystone-secret", - Namespace: "default", - }, - Data: map[string][]byte{ - "url": []byte(keystoneServer.URL), - "availability": []byte("public"), - "username": []byte("test-user"), - "password": []byte("test-password"), - "projectName": []byte("test-project"), - "userDomainName": []byte("default"), - "projectDomainName": []byte("default"), - }, - }) - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - Build() - config := HistoryCleanupConfig{ - KeystoneSecretRef: corev1.SecretReference{ - Name: "keystone-secret", - Namespace: "default", - }, - } - err := HistoryCleanup(context.Background(), client, config) - - if tt.expectError && err == nil { - t.Error("Expected error but got none") - } - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - - if !tt.expectError { - // Verify expected history entries were deleted - for _, expectedDeleted := range tt.expectedDeleted { - var history v1alpha1.History - err := client.Get(context.Background(), - types.NamespacedName{Name: expectedDeleted}, &history) - if err == nil { - t.Errorf("Expected history %s to be deleted but it still exists", expectedDeleted) - } - } - - // Verify other history entries still exist - for _, originalHistory := range tt.histories { - shouldBeDeleted := false - for _, expectedDeleted := range tt.expectedDeleted { - if originalHistory.Name == expectedDeleted { - shouldBeDeleted = true - break - } - } - if !shouldBeDeleted { - var history v1alpha1.History - err := client.Get(context.Background(), - types.NamespacedName{Name: originalHistory.Name}, &history) - if err != nil { - t.Errorf("Expected history %s to still exist but got error: %v", - originalHistory.Name, err) - } - } - } - } - }) - } -} - -func TestCleanupCinderHistoryCancel(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add corev1 scheme: %v", err) - } - - objects := []client.Object{ - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "keystone-secret", - Namespace: "default", - }, - Data: map[string][]byte{ - "url": []byte("http://invalid-keystone-url"), - "availability": []byte("public"), - "username": []byte("test-user"), - "password": []byte("test-password"), - "projectName": []byte("test-project"), - "userDomainName": []byte("default"), - "projectDomainName": []byte("default"), - }, - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - Build() - - config := HistoryCleanupConfig{ - KeystoneSecretRef: corev1.SecretReference{ - Name: "keystone-secret", - Namespace: "default", - }, - } - - ctx, cancel := context.WithTimeout(context.Background(), 0) - defer cancel() - - // This should exit quickly due to context cancellation - if err := HistoryCleanup(ctx, client, config); err != nil { - if !errors.Is(err, context.DeadlineExceeded) { - t.Errorf("Unexpected error during cleanup: %v", err) - } - } - - // If we reach here without hanging, the test passed -} - -type mockVolume struct { - ID string `json:"id"` -} diff --git a/internal/scheduling/lib/explanation.go b/internal/scheduling/lib/explanation.go new file mode 100644 index 000000000..3506f8a54 --- /dev/null +++ b/internal/scheduling/lib/explanation.go @@ -0,0 +1,108 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package lib + +import ( + "fmt" + "sort" + "strings" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +// maxHostsInExplanation caps how many host names are listed per line in a +// generated explanation before the remainder is summarized as a count. +const maxHostsInExplanation = 10 + +// joinHostsCapped joins up to max host names. If hosts exceeds max, it appends +// a count of the omitted entries, e.g. "host-a, host-b (and 48 more)". +func joinHostsCapped(hosts []string, maxHosts int) string { + if len(hosts) <= maxHosts { + return strings.Join(hosts, ", ") + } + return fmt.Sprintf("%s (and %d more)", strings.Join(hosts[:maxHosts], ", "), len(hosts)-maxHosts) +} + +// GenerateExplanation produces a human-readable explanation from a decision +// result. On failure it includes the error. On success it describes which +// pipeline steps filtered out which hosts and, for the surviving hosts, how the +// weighers ranked them (via ExplainWeighing). The result is written to +// Decision.Status.Explanation so it propagates to the scheduled resource (e.g. +// the VM CortexSpec.Explanation). +func GenerateExplanation(result *v1alpha1.DecisionResult, pipelineErr error) string { + if pipelineErr != nil { + return fmt.Sprintf("Pipeline run failed: %s.", pipelineErr.Error()) + } + + if result == nil || len(result.StepResults) == 0 { + if result != nil && result.TargetHost != nil { + return fmt.Sprintf("Selected host: %s.", *result.TargetHost) + } + return "" + } + + // Get all initial hosts from input weights. + var allHosts map[string]float64 + if len(result.RawInWeights) > 0 { + allHosts = result.RawInWeights + } else if len(result.NormalizedInWeights) > 0 { + allHosts = result.NormalizedInWeights + } + if allHosts == nil { + if result.TargetHost != nil { + return fmt.Sprintf("Selected host: %s.", *result.TargetHost) + } + return "" + } + + var sb strings.Builder + fmt.Fprintf(&sb, "Started with %d host(s).\n\n", len(allHosts)) + + // Track current set of surviving hosts. + currentHosts := make(map[string]bool, len(allHosts)) + for h := range allHosts { + currentHosts[h] = true + } + + for _, step := range result.StepResults { + // Determine which hosts were removed by this step. + var removed []string + for h := range currentHosts { + if _, exists := step.Activations[h]; !exists { + removed = append(removed, h) + } + } + if len(removed) > 0 { + sort.Strings(removed) + for _, h := range removed { + delete(currentHosts, h) + } + fmt.Fprintf(&sb, "%s filtered out %s\n", + step.StepName, + joinHostsCapped(removed, maxHostsInExplanation), + ) + } + } + + // Summary of remaining hosts. + remaining := make([]string, 0, len(currentHosts)) + for h := range currentHosts { + remaining = append(remaining, h) + } + sort.Strings(remaining) + fmt.Fprintf(&sb, "\n%d hosts remaining (%s)\n", + len(remaining), + joinHostsCapped(remaining, maxHostsInExplanation), + ) + + if result.TargetHost != nil { + fmt.Fprintf(&sb, "\nSelected host: %s.", *result.TargetHost) + } + + if weighingExpl := ExplainWeighing(result); weighingExpl != "" { + fmt.Fprintf(&sb, "\n\n%s", weighingExpl) + } + + return strings.TrimSpace(sb.String()) +} diff --git a/internal/scheduling/lib/explanation_test.go b/internal/scheduling/lib/explanation_test.go new file mode 100644 index 000000000..25fcf075f --- /dev/null +++ b/internal/scheduling/lib/explanation_test.go @@ -0,0 +1,169 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package lib + +import ( + "errors" + "fmt" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +func ptr[T any](v T) *T { return &v } + +func TestGenerateExplanation(t *testing.T) { + tests := []struct { + name string + result *v1alpha1.DecisionResult + err error + expected string + }{ + { + name: "nil result no error", + result: nil, + err: nil, + expected: "", + }, + { + name: "nil result with error", + result: nil, + err: errors.New("something broke"), + expected: "Pipeline run failed: something broke.", + }, + { + name: "result with target host only no steps", + result: &v1alpha1.DecisionResult{ + TargetHost: ptr("host-1"), + }, + expected: "Selected host: host-1.", + }, + { + name: "result with RawInWeights and filtering steps", + result: &v1alpha1.DecisionResult{ + RawInWeights: map[string]float64{ + "host-a": 1.0, + "host-b": 0.5, + "host-c": 0.3, + }, + StepResults: []v1alpha1.StepResult{ + { + StepName: "filter_capacity", + Activations: map[string]float64{ + "host-a": 1.0, + "host-b": 0.5, + }, + }, + { + StepName: "filter_status", + Activations: map[string]float64{ + "host-a": 1.0, + }, + }, + }, + TargetHost: ptr("host-a"), + }, + expected: "Started with 3 host(s).\n\n" + + "filter_capacity filtered out host-c\n" + + "filter_status filtered out host-b\n\n" + + "1 hosts remaining (host-a)\n\n" + + "Selected host: host-a.", + }, + { + name: "uses NormalizedInWeights when RawInWeights empty", + result: &v1alpha1.DecisionResult{ + NormalizedInWeights: map[string]float64{ + "host-x": 0.6, + "host-y": 0.4, + }, + StepResults: []v1alpha1.StepResult{ + { + StepName: "weigher_cpu", + Activations: map[string]float64{ + "host-x": 0.8, + "host-y": 0.2, + }, + }, + }, + TargetHost: ptr("host-x"), + }, + expected: "Started with 2 host(s).\n\n\n2 hosts remaining (host-x, host-y)\n\nSelected host: host-x.", + }, + { + name: "no weights no target", + result: &v1alpha1.DecisionResult{ + StepResults: []v1alpha1.StepResult{ + {StepName: "some-step", Activations: map[string]float64{}}, + }, + }, + expected: "", + }, + { + name: "no weights with target", + result: &v1alpha1.DecisionResult{ + TargetHost: ptr("host-1"), + StepResults: []v1alpha1.StepResult{ + {StepName: "some-step", Activations: map[string]float64{}}, + }, + }, + expected: "Selected host: host-1.", + }, + { + name: "all hosts survive all steps", + result: &v1alpha1.DecisionResult{ + RawInWeights: map[string]float64{ + "host-a": 1.0, + "host-b": 0.5, + }, + StepResults: []v1alpha1.StepResult{ + { + StepName: "noop", + Activations: map[string]float64{ + "host-a": 1.0, + "host-b": 0.5, + }, + }, + }, + TargetHost: ptr("host-a"), + }, + expected: "Started with 2 host(s).\n\n\n2 hosts remaining (host-a, host-b)\n\nSelected host: host-a.", + }, + { + name: "large cluster caps host lists", + result: func() *v1alpha1.DecisionResult { + weights := make(map[string]float64, 50) + for i := range 50 { + weights[fmt.Sprintf("host-%03d", i)] = 1.0 + } + // Step filters out 20 hosts (keep host-000..host-029). + surviving := make(map[string]float64, 30) + for i := range 30 { + surviving[fmt.Sprintf("host-%03d", i)] = 1.0 + } + return &v1alpha1.DecisionResult{ + RawInWeights: weights, + StepResults: []v1alpha1.StepResult{ + { + StepName: "filter_big", + Activations: surviving, + }, + }, + TargetHost: ptr("host-000"), + } + }(), + expected: "Started with 50 host(s).\n\n" + + "filter_big filtered out host-030, host-031, host-032, host-033, host-034, host-035, host-036, host-037, host-038, host-039 (and 10 more)\n\n" + + "30 hosts remaining (host-000, host-001, host-002, host-003, host-004, host-005, host-006, host-007, host-008, host-009 (and 20 more))\n\n" + + "Selected host: host-000.", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GenerateExplanation(tt.result, tt.err) + if got != tt.expected { + t.Errorf("GenerateExplanation() =\n%q\nwant:\n%q", got, tt.expected) + } + }) + } +} diff --git a/internal/scheduling/lib/history_client.go b/internal/scheduling/lib/history_client.go deleted file mode 100644 index 035ec92b4..000000000 --- a/internal/scheduling/lib/history_client.go +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package lib - -import ( - "context" - "errors" - "fmt" - "sort" - "strings" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/events" - "k8s.io/client-go/util/retry" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - maxHostsInExplanation = 10 - maxHistoryEntries = 10 - maxHostsInOrderedList = 3 -) - -// joinHostsCapped joins up to max host names. If hosts exceeds max, it appends -// a count of the omitted entries, e.g. "host-a, host-b (and 48 more)". -func joinHostsCapped(hosts []string, maxHosts int) string { - if len(hosts) <= maxHosts { - return strings.Join(hosts, ", ") - } - return fmt.Sprintf("%s (and %d more)", strings.Join(hosts[:maxHosts], ", "), len(hosts)-maxHosts) -} - -func getName(schedulingDomain v1alpha1.SchedulingDomain, resourceID string) string { - return fmt.Sprintf("%s-%s", schedulingDomain, resourceID) -} - -// generateSimplifiedExplanation produces a human-readable explanation from a -// decision result. On failure it includes the error. On success it describes -// which pipeline steps filtered out which hosts. -func generateExplanation(result *v1alpha1.DecisionResult, pipelineErr error) string { - if pipelineErr != nil { - return fmt.Sprintf("Pipeline run failed: %s.", pipelineErr.Error()) - } - - if result == nil || len(result.StepResults) == 0 { - if result != nil && result.TargetHost != nil { - return fmt.Sprintf("Selected host: %s.", *result.TargetHost) - } - return "" - } - - // Get all initial hosts from input weights. - var allHosts map[string]float64 - if len(result.RawInWeights) > 0 { - allHosts = result.RawInWeights - } else if len(result.NormalizedInWeights) > 0 { - allHosts = result.NormalizedInWeights - } - if allHosts == nil { - if result.TargetHost != nil { - return fmt.Sprintf("Selected host: %s.", *result.TargetHost) - } - return "" - } - - var sb strings.Builder - fmt.Fprintf(&sb, "Started with %d host(s).\n\n", len(allHosts)) - - // Track current set of surviving hosts. - currentHosts := make(map[string]bool, len(allHosts)) - for h := range allHosts { - currentHosts[h] = true - } - - for _, step := range result.StepResults { - // Determine which hosts were removed by this step. - var removed []string - for h := range currentHosts { - if _, exists := step.Activations[h]; !exists { - removed = append(removed, h) - } - } - if len(removed) > 0 { - sort.Strings(removed) - for _, h := range removed { - delete(currentHosts, h) - } - fmt.Fprintf(&sb, "%s filtered out %s\n", - step.StepName, - joinHostsCapped(removed, maxHostsInExplanation), - ) - } - } - - // Summary of remaining hosts. - remaining := make([]string, 0, len(currentHosts)) - for h := range currentHosts { - remaining = append(remaining, h) - } - sort.Strings(remaining) - fmt.Fprintf(&sb, "\n%d hosts remaining (%s)\n", - len(remaining), - joinHostsCapped(remaining, maxHostsInExplanation), - ) - - if result.TargetHost != nil { - fmt.Fprintf(&sb, "\nSelected host: %s.", *result.TargetHost) - } - - if weighingExpl := ExplainWeighing(result); weighingExpl != "" { - fmt.Fprintf(&sb, "\n\n%s", weighingExpl) - } - - return strings.TrimSpace(sb.String()) -} - -// HistoryClient manages History CRDs for scheduling decisions. It holds the -// Kubernetes client and event recorder so callers don't have to pass them on -// every invocation. -type HistoryClient struct { - Client client.Client - Recorder events.EventRecorder -} - -// CreateOrUpdateHistory creates or updates a History CRD for the given decision. -// It is called after every pipeline run (success and failure). The pipelineErr -// parameter is used to generate a meaningful explanation when the pipeline fails. -// If a non-nil Recorder is set, a Kubernetes Event is emitted on the History -// object to short-term persist the scheduling decision. -func (h *HistoryClient) CreateOrUpdateHistory( - ctx context.Context, - decision *v1alpha1.Decision, - az *string, - pipelineErr error, -) error { - - if decision == nil { - return errors.New("decision cannot be nil") - } - - log := ctrl.LoggerFrom(ctx) - - name := getName(decision.Spec.SchedulingDomain, decision.Spec.ResourceID) - - history := &v1alpha1.History{} - err := h.Client.Get(ctx, client.ObjectKey{Name: name}, history) - - if apierrors.IsNotFound(err) { - // Create new History CRD. - history = &v1alpha1.History{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: decision.Spec.SchedulingDomain, - ResourceID: decision.Spec.ResourceID, - AvailabilityZone: az, - }, - } - if createErr := h.Client.Create(ctx, history); createErr != nil { - if apierrors.IsAlreadyExists(createErr) { - // Another controller beat us to it, re-fetch. - if getErr := h.Client.Get(ctx, client.ObjectKey{Name: name}, history); getErr != nil { - return getErr - } - } else { - log.Error(createErr, "failed to create history CRD", "name", name) - return createErr - } - } - } else if err != nil { - log.Error(err, "failed to get history CRD", "name", name) - return err - } - - successful := pipelineErr == nil && decision.Status.Result != nil && decision.Status.Result.TargetHost != nil - - namespacedName := client.ObjectKey{Name: name} - - // Use Update instead of MergeFrom+Patch because JSON merge patch strips - // boolean false values, which causes CRD validation to reject the patch - // when Successful is false. Retry on conflict to handle concurrent updates. - firstAttempt := true - if retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { - // On retries, re-fetch the latest History object after a conflict. - if !firstAttempt { - if getErr := h.Client.Get(ctx, namespacedName, history); getErr != nil { - return getErr - } - } - firstAttempt = false - - // Archive the previous current decision into the history list. - if !history.Status.Current.Timestamp.IsZero() { - orderedHosts := history.Status.Current.OrderedHosts - if orderedHosts == nil { - orderedHosts = []string{} - } - entry := v1alpha1.SchedulingHistoryEntry{ - Timestamp: history.Status.Current.Timestamp, - PipelineRef: history.Status.Current.PipelineRef, - Intent: history.Status.Current.Intent, - OrderedHosts: orderedHosts, - Successful: history.Status.Current.Successful, - } - history.Status.History = append(history.Status.History, entry) - if len(history.Status.History) > maxHistoryEntries { - history.Status.History = history.Status.History[len(history.Status.History)-maxHistoryEntries:] - } - } - - // Build the new current decision. - current := v1alpha1.CurrentDecision{ - Timestamp: metav1.Now(), - PipelineRef: decision.Spec.PipelineRef, - Intent: decision.Spec.Intent, - Successful: successful, - Explanation: generateExplanation(decision.Status.Result, pipelineErr), - } - - current.OrderedHosts = []string{} - if decision.Status.Result != nil { - current.TargetHost = decision.Status.Result.TargetHost - hosts := decision.Status.Result.OrderedHosts - if len(hosts) > maxHostsInOrderedList { - hosts = hosts[:maxHostsInOrderedList] - } - current.OrderedHosts = hosts - } - history.Status.Current = current - - // Set Ready condition — True only when a host was successfully selected. - condStatus := metav1.ConditionTrue - reason := v1alpha1.HistoryReasonSchedulingSucceeded - message := "scheduling decision selected a target host" - if pipelineErr != nil { - condStatus = metav1.ConditionFalse - reason = v1alpha1.HistoryReasonPipelineRunFailed - message = "pipeline run failed: " + pipelineErr.Error() - } else if !successful { - condStatus = metav1.ConditionFalse - reason = v1alpha1.HistoryReasonNoHostFound - message = "pipeline completed but no suitable host was found" - } - meta.SetStatusCondition(&history.Status.Conditions, metav1.Condition{ - Type: v1alpha1.HistoryConditionReady, - Status: condStatus, - Reason: reason, - Message: message, - }) - - return h.Client.Status().Update(ctx, history) - }); retryErr != nil { - log.Error(retryErr, "failed to update history CRD status", "name", name) - return retryErr - } - - // Emit a Kubernetes Event on the History object to short-term persist the - // scheduling decision. Events auto-expire (default TTL ~1h) so this gives - // devops short-term visibility into individual scheduling runs. - if h.Recorder != nil { - eventType := corev1.EventTypeNormal - eventReason := v1alpha1.HistoryReasonSchedulingSucceeded - action := "Scheduled" - if !successful { - eventType = corev1.EventTypeWarning - eventReason = "SchedulingFailed" - action = "FailedScheduling" - } - h.Recorder.Eventf(history, nil, eventType, eventReason, action, "%s", history.Status.Current.Explanation) - } - - log.Info("history CRD updated", "name", name, "entries", len(history.Status.History)) - return nil -} - -// Delete deletes the History CRD associated with the given scheduling domain -// and resource ID. It is a no-op if the History CRD does not exist. -func (h *HistoryClient) Delete( - ctx context.Context, - schedulingDomain v1alpha1.SchedulingDomain, - resourceID string, -) error { - - log := ctrl.LoggerFrom(ctx) - name := getName(schedulingDomain, resourceID) - - history := &v1alpha1.History{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - } - if err := h.Client.Delete(ctx, history); err != nil { - if apierrors.IsNotFound(err) { - return nil - } - log.Error(err, "failed to delete history CRD", "name", name) - return err - } - log.Info("deleted history CRD", "name", name) - return nil -} diff --git a/internal/scheduling/lib/history_client_test.go b/internal/scheduling/lib/history_client_test.go deleted file mode 100644 index 1e7f63559..000000000 --- a/internal/scheduling/lib/history_client_test.go +++ /dev/null @@ -1,631 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package lib - -import ( - "context" - "errors" - "fmt" - "strings" - "testing" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestGetName(t *testing.T) { - tests := []struct { - domain v1alpha1.SchedulingDomain - resourceID string - expected string - }{ - {v1alpha1.SchedulingDomainNova, "uuid-1", "nova-uuid-1"}, - {v1alpha1.SchedulingDomainCinder, "vol-abc", "cinder-vol-abc"}, - {v1alpha1.SchedulingDomainManila, "share-xyz", "manila-share-xyz"}, - {v1alpha1.SchedulingDomainMachines, "machine-1", "machines-machine-1"}, - {v1alpha1.SchedulingDomainPods, "pod-ns-name", "pods-pod-ns-name"}, - } - for _, tt := range tests { - t.Run(tt.expected, func(t *testing.T) { - got := getName(tt.domain, tt.resourceID) - if got != tt.expected { - t.Errorf("getName(%q, %q) = %q, want %q", tt.domain, tt.resourceID, got, tt.expected) - } - }) - } -} - -func TestGenerateExplanation(t *testing.T) { - tests := []struct { - name string - result *v1alpha1.DecisionResult - err error - expected string - }{ - { - name: "nil result no error", - result: nil, - err: nil, - expected: "", - }, - { - name: "nil result with error", - result: nil, - err: errors.New("something broke"), - expected: "Pipeline run failed: something broke.", - }, - { - name: "result with target host only no steps", - result: &v1alpha1.DecisionResult{ - TargetHost: new("host-1"), - }, - expected: "Selected host: host-1.", - }, - { - name: "result with RawInWeights and filtering steps", - result: &v1alpha1.DecisionResult{ - RawInWeights: map[string]float64{ - "host-a": 1.0, - "host-b": 0.5, - "host-c": 0.3, - }, - StepResults: []v1alpha1.StepResult{ - { - StepName: "filter_capacity", - Activations: map[string]float64{ - "host-a": 1.0, - "host-b": 0.5, - }, - }, - { - StepName: "filter_status", - Activations: map[string]float64{ - "host-a": 1.0, - }, - }, - }, - TargetHost: new("host-a"), - }, - expected: "Started with 3 host(s).\n\n" + - "filter_capacity filtered out host-c\n" + - "filter_status filtered out host-b\n\n" + - "1 hosts remaining (host-a)\n\n" + - "Selected host: host-a.", - }, - { - name: "uses NormalizedInWeights when RawInWeights empty", - result: &v1alpha1.DecisionResult{ - NormalizedInWeights: map[string]float64{ - "host-x": 0.6, - "host-y": 0.4, - }, - StepResults: []v1alpha1.StepResult{ - { - StepName: "weigher_cpu", - Activations: map[string]float64{ - "host-x": 0.8, - "host-y": 0.2, - }, - }, - }, - TargetHost: new("host-x"), - }, - expected: "Started with 2 host(s).\n\n\n2 hosts remaining (host-x, host-y)\n\nSelected host: host-x.", - }, - { - name: "no weights no target", - result: &v1alpha1.DecisionResult{ - StepResults: []v1alpha1.StepResult{ - {StepName: "some-step", Activations: map[string]float64{}}, - }, - }, - expected: "", - }, - { - name: "no weights with target", - result: &v1alpha1.DecisionResult{ - TargetHost: new("host-1"), - StepResults: []v1alpha1.StepResult{ - {StepName: "some-step", Activations: map[string]float64{}}, - }, - }, - expected: "Selected host: host-1.", - }, - { - name: "all hosts survive all steps", - result: &v1alpha1.DecisionResult{ - RawInWeights: map[string]float64{ - "host-a": 1.0, - "host-b": 0.5, - }, - StepResults: []v1alpha1.StepResult{ - { - StepName: "noop", - Activations: map[string]float64{ - "host-a": 1.0, - "host-b": 0.5, - }, - }, - }, - TargetHost: new("host-a"), - }, - expected: "Started with 2 host(s).\n\n\n2 hosts remaining (host-a, host-b)\n\nSelected host: host-a.", - }, - { - name: "large cluster caps host lists", - result: func() *v1alpha1.DecisionResult { - weights := make(map[string]float64, 50) - for i := range 50 { - weights[fmt.Sprintf("host-%03d", i)] = 1.0 - } - // Step filters out 20 hosts (keep host-000..host-029). - surviving := make(map[string]float64, 30) - for i := range 30 { - surviving[fmt.Sprintf("host-%03d", i)] = 1.0 - } - return &v1alpha1.DecisionResult{ - RawInWeights: weights, - StepResults: []v1alpha1.StepResult{ - { - StepName: "filter_big", - Activations: surviving, - }, - }, - TargetHost: new("host-000"), - } - }(), - expected: "Started with 50 host(s).\n\n" + - "filter_big filtered out host-030, host-031, host-032, host-033, host-034, host-035, host-036, host-037, host-038, host-039 (and 10 more)\n\n" + - "30 hosts remaining (host-000, host-001, host-002, host-003, host-004, host-005, host-006, host-007, host-008, host-009 (and 20 more))\n\n" + - "Selected host: host-000.", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := generateExplanation(tt.result, tt.err) - if got != tt.expected { - t.Errorf("generateExplanation() =\n%q\nwant:\n%q", got, tt.expected) - } - }) - } -} - -func newTestScheme(t *testing.T) *runtime.Scheme { - t.Helper() - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - return scheme -} - -func TestHistoryClient_CreateOrUpdateHistory(t *testing.T) { - tests := []struct { - name string - // setup returns a fake client and any pre-existing objects. - setup func(t *testing.T) client.Client - // decision to upsert. - decision *v1alpha1.Decision - pipelineErr error - // assertions on the history list (archived entries only). - expectHistoryLen int - // assertions on the current decision. - expectTargetHost *string - expectSuccessful bool - expectCondStatus metav1.ConditionStatus - expectReason string - // expectMessageContains, if set, asserts the Ready condition message - // surfaces this substring (used to check the pipeline-error path tells - // the user why scheduling failed). - expectMessageContains string - checkExplanation func(t *testing.T, explanation string) - checkCurrentHosts func(t *testing.T, hosts []string) - }{ - { - name: "create new history", - setup: func(t *testing.T) client.Client { - return fake.NewClientBuilder(). - WithScheme(newTestScheme(t)). - WithStatusSubresource(&v1alpha1.History{}). - Build() - }, - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "uuid-1", - PipelineRef: corev1.ObjectReference{Name: "nova-pipeline"}, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: new("compute-1"), - }, - }, - }, - expectHistoryLen: 0, - expectTargetHost: new("compute-1"), - expectSuccessful: true, - expectCondStatus: metav1.ConditionTrue, - expectReason: v1alpha1.HistoryReasonSchedulingSucceeded, - checkExplanation: func(t *testing.T, explanation string) { - if !strings.Contains(explanation, "Selected host: compute-1") { - t.Errorf("expected explanation to contain selected host, got: %q", explanation) - } - }, - }, - { - name: "update existing history with pre-existing entries", - setup: func(t *testing.T) client.Client { - scheme := newTestScheme(t) - existing := &v1alpha1.History{ - ObjectMeta: metav1.ObjectMeta{Name: "nova-uuid-2"}, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "uuid-2", - }, - Status: v1alpha1.HistoryStatus{ - History: []v1alpha1.SchedulingHistoryEntry{ - {Timestamp: metav1.Now(), PipelineRef: corev1.ObjectReference{Name: "old-pipeline"}}, - }, - }, - } - return fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(existing). - WithStatusSubresource(&v1alpha1.History{}). - Build() - }, - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "uuid-2", - PipelineRef: corev1.ObjectReference{Name: "nova-pipeline"}, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: new("compute-2"), - }, - }, - }, - expectHistoryLen: 1, // pre-existing entry preserved, no current to archive - expectTargetHost: new("compute-2"), - expectSuccessful: true, - expectCondStatus: metav1.ConditionTrue, - expectReason: v1alpha1.HistoryReasonSchedulingSucceeded, - }, - { - name: "archive current to history on second upsert", - setup: func(t *testing.T) client.Client { - scheme := newTestScheme(t) - existing := &v1alpha1.History{ - ObjectMeta: metav1.ObjectMeta{Name: "nova-uuid-3"}, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "uuid-3", - }, - Status: v1alpha1.HistoryStatus{ - Current: v1alpha1.CurrentDecision{ - Timestamp: metav1.Now(), - PipelineRef: corev1.ObjectReference{Name: "old-pipeline"}, - Intent: v1alpha1.SchedulingIntentUnknown, - Successful: true, - TargetHost: new("old-host"), - }, - }, - } - return fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(existing). - WithStatusSubresource(&v1alpha1.History{}). - Build() - }, - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "uuid-3", - PipelineRef: corev1.ObjectReference{Name: "new-pipeline"}, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: new("new-host"), - }, - }, - }, - expectHistoryLen: 1, // old current archived - expectTargetHost: new("new-host"), - expectSuccessful: true, - expectCondStatus: metav1.ConditionTrue, - expectReason: v1alpha1.HistoryReasonSchedulingSucceeded, - }, - { - name: "pipeline error", - setup: func(t *testing.T) client.Client { - return fake.NewClientBuilder(). - WithScheme(newTestScheme(t)). - WithStatusSubresource(&v1alpha1.History{}). - Build() - }, - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - ResourceID: "vol-1", - PipelineRef: corev1.ObjectReference{Name: "cinder-pipeline"}, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - }, - pipelineErr: errors.New("no hosts available"), - expectHistoryLen: 0, - expectTargetHost: nil, - expectSuccessful: false, - expectCondStatus: metav1.ConditionFalse, - expectReason: v1alpha1.HistoryReasonPipelineRunFailed, - // The Ready condition message must tell the user the pipeline - // errored and why, so devops can troubleshoot from the CRD status. - expectMessageContains: "pipeline run failed: no hosts available", - checkExplanation: func(t *testing.T, explanation string) { - if !strings.Contains(explanation, "no hosts available") { - t.Errorf("expected explanation to contain error text, got: %q", explanation) - } - }, - }, - { - name: "no host found", - setup: func(t *testing.T) client.Client { - return fake.NewClientBuilder(). - WithScheme(newTestScheme(t)). - WithStatusSubresource(&v1alpha1.History{}). - Build() - }, - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "uuid-nohost", - PipelineRef: corev1.ObjectReference{Name: "nova-pipeline"}, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - // Pipeline succeeded but no target host selected (all filtered out). - OrderedHosts: []string{}, - }, - }, - }, - expectHistoryLen: 0, - expectTargetHost: nil, - expectSuccessful: false, - expectCondStatus: metav1.ConditionFalse, - expectReason: v1alpha1.HistoryReasonNoHostFound, - }, - { - name: "history capped at 10 entries", - setup: func(t *testing.T) client.Client { - scheme := newTestScheme(t) - entries := make([]v1alpha1.SchedulingHistoryEntry, 10) - for i := range entries { - entries[i] = v1alpha1.SchedulingHistoryEntry{ - Timestamp: metav1.Now(), - PipelineRef: corev1.ObjectReference{Name: fmt.Sprintf("pipeline-%d", i)}, - } - } - existing := &v1alpha1.History{ - ObjectMeta: metav1.ObjectMeta{Name: "manila-share-1"}, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - ResourceID: "share-1", - }, - Status: v1alpha1.HistoryStatus{ - Current: v1alpha1.CurrentDecision{ - Timestamp: metav1.Now(), - PipelineRef: corev1.ObjectReference{Name: "prev-pipeline"}, - Successful: true, - TargetHost: new("old-backend"), - }, - History: entries, - }, - } - return fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(existing). - WithStatusSubresource(&v1alpha1.History{}). - Build() - }, - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - ResourceID: "share-1", - PipelineRef: corev1.ObjectReference{Name: "manila-pipeline"}, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: new("backend-1"), - }, - }, - }, - expectHistoryLen: 10, // 10 existing + 1 archived current, capped to 10 - expectTargetHost: new("backend-1"), - expectSuccessful: true, - expectCondStatus: metav1.ConditionTrue, - expectReason: v1alpha1.HistoryReasonSchedulingSucceeded, - }, - { - name: "ordered hosts capped at 3", - setup: func(t *testing.T) client.Client { - return fake.NewClientBuilder(). - WithScheme(newTestScheme(t)). - WithStatusSubresource(&v1alpha1.History{}). - Build() - }, - decision: &v1alpha1.Decision{ - Spec: v1alpha1.DecisionSpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "uuid-cap", - PipelineRef: corev1.ObjectReference{Name: "nova-pipeline"}, - Intent: v1alpha1.SchedulingIntentUnknown, - }, - Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{ - TargetHost: new("h1"), - OrderedHosts: []string{"h1", "h2", "h3", "h4", "h5"}, - }, - }, - }, - expectHistoryLen: 0, - expectTargetHost: new("h1"), - expectSuccessful: true, - expectCondStatus: metav1.ConditionTrue, - expectReason: v1alpha1.HistoryReasonSchedulingSucceeded, - checkCurrentHosts: func(t *testing.T, hosts []string) { - if len(hosts) != 3 { - t.Errorf("ordered hosts length = %d, want 3", len(hosts)) - } - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := tt.setup(t) - hm := HistoryClient{Client: c} - - err := hm.CreateOrUpdateHistory(context.Background(), tt.decision, nil, tt.pipelineErr) - if err != nil { - t.Fatalf("Upsert() returned error: %v", err) - } - - // Fetch the history CRD. - name := getName(tt.decision.Spec.SchedulingDomain, tt.decision.Spec.ResourceID) - var history v1alpha1.History - if err := c.Get(context.Background(), client.ObjectKey{Name: name}, &history); err != nil { - t.Fatalf("Failed to get history: %v", err) - } - - // Verify spec. - if history.Spec.SchedulingDomain != tt.decision.Spec.SchedulingDomain { - t.Errorf("spec.schedulingDomain = %q, want %q", history.Spec.SchedulingDomain, tt.decision.Spec.SchedulingDomain) - } - if history.Spec.ResourceID != tt.decision.Spec.ResourceID { - t.Errorf("spec.resourceID = %q, want %q", history.Spec.ResourceID, tt.decision.Spec.ResourceID) - } - - // Verify history list length. - if len(history.Status.History) != tt.expectHistoryLen { - t.Errorf("status.history has %d entries, want %d", len(history.Status.History), tt.expectHistoryLen) - } - - // Verify current decision. - current := history.Status.Current - if current.Successful != tt.expectSuccessful { - t.Errorf("current.successful = %v, want %v", current.Successful, tt.expectSuccessful) - } - if current.PipelineRef.Name != tt.decision.Spec.PipelineRef.Name { - t.Errorf("current.pipelineRef = %q, want %q", current.PipelineRef.Name, tt.decision.Spec.PipelineRef.Name) - } - - // Verify target host. - if tt.expectTargetHost == nil { - if current.TargetHost != nil { - t.Errorf("current.targetHost = %q, want nil", *current.TargetHost) - } - } else { - if current.TargetHost == nil { - t.Errorf("current.targetHost = nil, want %q", *tt.expectTargetHost) - } else if *current.TargetHost != *tt.expectTargetHost { - t.Errorf("current.targetHost = %q, want %q", *current.TargetHost, *tt.expectTargetHost) - } - } - - // Verify condition. - if len(history.Status.Conditions) == 0 { - t.Fatal("expected at least one condition") - } - readyCond := history.Status.Conditions[0] - if readyCond.Status != tt.expectCondStatus { - t.Errorf("condition status = %q, want %q", readyCond.Status, tt.expectCondStatus) - } - if readyCond.Reason != tt.expectReason { - t.Errorf("condition reason = %q, want %q", readyCond.Reason, tt.expectReason) - } - if tt.expectMessageContains != "" && !strings.Contains(readyCond.Message, tt.expectMessageContains) { - t.Errorf("condition message = %q, want it to contain %q", readyCond.Message, tt.expectMessageContains) - } - - // Verify explanation. - if tt.checkExplanation != nil { - tt.checkExplanation(t, current.Explanation) - } - - // Verify ordered hosts on current. - if tt.checkCurrentHosts != nil { - tt.checkCurrentHosts(t, current.OrderedHosts) - } - }) - } -} - -func TestHistoryClient_Delete(t *testing.T) { - tests := []struct { - name string - domain v1alpha1.SchedulingDomain - resourceID string - preCreate bool - }{ - { - name: "delete existing", - domain: v1alpha1.SchedulingDomainNova, - resourceID: "uuid-del", - preCreate: true, - }, - { - name: "delete non-existing is no-op", - domain: v1alpha1.SchedulingDomainCinder, - resourceID: "does-not-exist", - preCreate: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - scheme := newTestScheme(t) - var objects []client.Object - if tt.preCreate { - objects = append(objects, &v1alpha1.History{ - ObjectMeta: metav1.ObjectMeta{ - Name: getName(tt.domain, tt.resourceID), - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: tt.domain, - ResourceID: tt.resourceID, - }, - }) - } - - c := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - Build() - hm := HistoryClient{Client: c} - - err := hm.Delete(context.Background(), tt.domain, tt.resourceID) - if err != nil { - t.Fatalf("Delete() returned error: %v", err) - } - - // Verify history is gone. - var histories v1alpha1.HistoryList - if err := c.List(context.Background(), &histories); err != nil { - t.Fatalf("Failed to list histories: %v", err) - } - if len(histories.Items) != 0 { - t.Errorf("expected 0 histories, got %d", len(histories.Items)) - } - }) - } -} diff --git a/internal/scheduling/lib/pipeline_controller.go b/internal/scheduling/lib/pipeline_controller.go index 25f994ea6..1cb4e5e06 100644 --- a/internal/scheduling/lib/pipeline_controller.go +++ b/internal/scheduling/lib/pipeline_controller.go @@ -33,8 +33,6 @@ type BasePipelineController[PipelineType any] struct { client.Client // The scheduling domain to scope resources to. SchedulingDomain v1alpha1.SchedulingDomain - // Manager for creating, updating, and deleting History CRDs. - HistoryManager HistoryClient } // GetPipeline returns the initialized pipeline for the given name, if it exists. diff --git a/internal/scheduling/machines/filter_weigher_pipeline_controller.go b/internal/scheduling/machines/filter_weigher_pipeline_controller.go index ebfddcc5c..3c542487b 100644 --- a/internal/scheduling/machines/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/machines/filter_weigher_pipeline_controller.go @@ -133,20 +133,16 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision return errors.New("no machine pools available for scheduling") } - // Execute the scheduling pipeline. Options not set: machine scheduling always records history. + // Execute the scheduling pipeline. request := ironcore.MachinePipelineRequest{Pools: pools.Items} result, err := pipeline.Run(request) - // Persist the result on the decision before upserting history so that - // CreateOrUpdateHistory can observe the target host and ordered hosts. - // On error the result is empty/meaningless, so only set it on success. if err == nil { decision.Status.Result = &result } - if !request.Options.SkipHistory { - if upsertErr := c.HistoryManager.CreateOrUpdateHistory(ctx, decision, nil, err); upsertErr != nil { - log.Error(upsertErr, "failed to create/update history") - } - } + // Record a human-readable explanation of the decision for downstream + // consumers. On error the result is nil and the explanation carries the + // failure reason. + decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) if err != nil { log.V(1).Error(err, "failed to run scheduler pipeline") return errors.New("failed to run scheduler pipeline") @@ -208,22 +204,12 @@ func (c *FilterWeigherPipelineController) handleMachine() handler.EventHandler { log.Error(err, "failed to process new machine for scheduling", "machine", newMachine.Name) } }, - DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { - c.processMu.Lock() - defer c.processMu.Unlock() - machine := evt.Object.(*ironcorev1alpha1.Machine) - if err := c.HistoryManager.Delete(ctx, v1alpha1.SchedulingDomainMachines, machine.Name); err != nil { - log := ctrl.LoggerFrom(ctx) - log.Error(err, "failed to delete history CRD for machine", "machine", machine.Name) - } - }, } } func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, mcl *multicluster.Client) error { c.Initializer = c c.SchedulingDomain = v1alpha1.SchedulingDomainMachines - c.HistoryManager = lib.HistoryClient{Client: mcl, Recorder: mcl.GetEventRecorder("cortex-machines-scheduler")} if err := mgr.Add(manager.RunnableFunc(c.InitAllPipelines)); err != nil { return err } diff --git a/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go b/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go index 9431f79ed..af85f6382 100644 --- a/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/machines/filter_weigher_pipeline_controller_test.go @@ -6,7 +6,6 @@ package machines import ( "context" "testing" - "time" "github.com/cobaltcore-dev/cortex/api/external/ironcore" ironcorev1alpha1 "github.com/cobaltcore-dev/cortex/api/external/ironcore/v1alpha1" @@ -125,7 +124,6 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { Pipelines: map[string]lib.FilterWeigherPipeline[ironcore.MachinePipelineRequest]{ "machines-scheduler": createMockPipeline(), }, - HistoryManager: lib.HistoryClient{Client: client}, }, Monitor: lib.FilterWeigherPipelineMonitor{}, } @@ -291,81 +289,10 @@ func TestFilterWeigherPipelineController_ProcessNewMachine(t *testing.T) { machine *ironcorev1alpha1.Machine machinePools []ironcorev1alpha1.MachinePool pipelineConfig *v1alpha1.Pipeline - createHistory bool expectError bool - expectHistoryCreated bool expectMachinePoolAssigned bool expectTargetHost string }{ - { - name: "successful machine processing with history creation", - machine: &ironcorev1alpha1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-machine", - Namespace: "default", - }, - Spec: ironcorev1alpha1.MachineSpec{ - Scheduler: "", - }, - }, - machinePools: []ironcorev1alpha1.MachinePool{ - { - ObjectMeta: metav1.ObjectMeta{Name: "pool1"}, - }, - { - ObjectMeta: metav1.ObjectMeta{Name: "pool2"}, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "machines-scheduler", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainMachines, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - createHistory: true, - expectError: false, - expectHistoryCreated: true, - expectMachinePoolAssigned: true, - expectTargetHost: "pool1", - }, - { - name: "successful machine processing without history creation", - machine: &ironcorev1alpha1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-machine-no-history", - Namespace: "default", - }, - Spec: ironcorev1alpha1.MachineSpec{ - Scheduler: "", - }, - }, - machinePools: []ironcorev1alpha1.MachinePool{ - { - ObjectMeta: metav1.ObjectMeta{Name: "pool1"}, - }, - }, - pipelineConfig: &v1alpha1.Pipeline{ - ObjectMeta: metav1.ObjectMeta{ - Name: "machines-scheduler", - }, - Spec: v1alpha1.PipelineSpec{ - Type: v1alpha1.PipelineTypeFilterWeigher, - SchedulingDomain: v1alpha1.SchedulingDomainMachines, - Filters: []v1alpha1.FilterSpec{}, - Weighers: []v1alpha1.WeigherSpec{}, - }, - }, - createHistory: false, - expectError: false, - expectHistoryCreated: true, - expectMachinePoolAssigned: true, - expectTargetHost: "pool1", - }, { name: "pipeline not configured", machine: &ironcorev1alpha1.Machine{ @@ -380,7 +307,6 @@ func TestFilterWeigherPipelineController_ProcessNewMachine(t *testing.T) { machinePools: []ironcorev1alpha1.MachinePool{}, pipelineConfig: nil, expectError: true, - expectHistoryCreated: false, expectMachinePoolAssigned: false, }, { @@ -406,9 +332,7 @@ func TestFilterWeigherPipelineController_ProcessNewMachine(t *testing.T) { Weighers: []v1alpha1.WeigherSpec{}, }, }, - createHistory: true, expectError: true, - expectHistoryCreated: false, expectMachinePoolAssigned: false, }, } @@ -426,14 +350,13 @@ func TestFilterWeigherPipelineController_ProcessNewMachine(t *testing.T) { client := fake.NewClientBuilder(). WithScheme(scheme). WithRuntimeObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}, &v1alpha1.History{}). + WithStatusSubresource(&v1alpha1.Decision{}). Build() controller := &FilterWeigherPipelineController{ BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[ironcore.MachinePipelineRequest]]{ Pipelines: map[string]lib.FilterWeigherPipeline[ironcore.MachinePipelineRequest]{}, PipelineConfigs: map[string]v1alpha1.Pipeline{}, - HistoryManager: lib.HistoryClient{Client: client}, }, Monitor: lib.FilterWeigherPipelineMonitor{}, } @@ -456,32 +379,6 @@ func TestFilterWeigherPipelineController_ProcessNewMachine(t *testing.T) { return } - // Check if history CRD was created when expected - if tt.expectHistoryCreated { - var histories v1alpha1.HistoryList - deadline := time.Now().Add(2 * time.Second) - for { - if err := client.List(context.Background(), &histories); err != nil { - t.Fatalf("Failed to list histories: %v", err) - } - if len(histories.Items) > 0 { - break - } - if time.Now().After(deadline) { - t.Fatal("timed out waiting for history CRD to be created") - } - time.Sleep(5 * time.Millisecond) - } - } else { - var histories v1alpha1.HistoryList - if err := client.List(context.Background(), &histories); err != nil { - t.Fatalf("Failed to list histories: %v", err) - } - if len(histories.Items) != 0 { - t.Error("Expected no history CRD but found one") - } - } - // Check if machine pool was assigned (if expected) if tt.expectMachinePoolAssigned { var updatedMachine ironcorev1alpha1.Machine diff --git a/internal/scheduling/manila/filter_weigher_pipeline_controller.go b/internal/scheduling/manila/filter_weigher_pipeline_controller.go index 347e14681..f6442f697 100644 --- a/internal/scheduling/manila/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/manila/filter_weigher_pipeline_controller.go @@ -113,17 +113,13 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision } result, err := pipeline.Run(request) - // Persist the result on the decision before upserting history so that - // CreateOrUpdateHistory can observe the target host and ordered hosts. - // On error the result is empty/meaningless, so only set it on success. if err == nil { decision.Status.Result = &result } - if !request.Options.SkipHistory { - if upsertErr := c.HistoryManager.CreateOrUpdateHistory(ctx, decision, nil, err); upsertErr != nil { - log.Error(upsertErr, "failed to create/update history") - } - } + // Record a human-readable explanation of the decision for downstream + // consumers. On error the result is nil and the explanation carries the + // failure reason. + decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) if err != nil { log.Error(err, "failed to run pipeline") return err @@ -149,7 +145,6 @@ func (c *FilterWeigherPipelineController) InitPipeline( func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, mcl *multicluster.Client) error { c.Initializer = c c.SchedulingDomain = v1alpha1.SchedulingDomainManila - c.HistoryManager = lib.HistoryClient{Client: mcl, Recorder: mcl.GetEventRecorder("cortex-manila-scheduler")} if err := mgr.Add(manager.RunnableFunc(c.InitAllPipelines)); err != nil { return err } diff --git a/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go b/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go index 11292d3f5..76dbd6f93 100644 --- a/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/manila/filter_weigher_pipeline_controller_test.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "testing" - "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -49,7 +48,7 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { }, Weights: map[string]float64{"manila-share-1@backend1": 1.0, "manila-share-2@backend2": 0.5}, Pipeline: "test-pipeline", - Options: scheduling.Options{SkipHistory: true}, + Options: scheduling.Options{}, } manilaRaw, err := json.Marshal(manilaRequest) @@ -247,13 +246,11 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) } tests := []struct { - name string - decision *v1alpha1.Decision - pipelineConfig *v1alpha1.Pipeline - createHistory bool - expectError bool - expectHistoryCreated bool - expectResult bool + name string + decision *v1alpha1.Decision + pipelineConfig *v1alpha1.Pipeline + expectError bool + expectResult bool }{ { name: "successful decision processing with creation", @@ -284,10 +281,8 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) Weighers: []v1alpha1.WeigherSpec{}, }, }, - createHistory: true, - expectError: false, - expectHistoryCreated: true, - expectResult: true, + expectError: false, + expectResult: true, }, { name: "successful decision processing without creation", @@ -317,10 +312,8 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) Weighers: []v1alpha1.WeigherSpec{}, }, }, - createHistory: false, - expectError: false, - expectHistoryCreated: false, - expectResult: true, + expectError: false, + expectResult: true, }, { name: "pipeline not configured", @@ -339,10 +332,9 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) }, }, }, - pipelineConfig: nil, - expectError: true, - expectHistoryCreated: false, - expectResult: false, + pipelineConfig: nil, + expectError: true, + expectResult: false, }, { name: "decision without manilaRaw spec", @@ -370,10 +362,8 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) Weighers: []v1alpha1.WeigherSpec{}, }, }, - createHistory: true, - expectError: true, - expectHistoryCreated: false, - expectResult: false, + expectError: true, + expectResult: false, }, } @@ -387,7 +377,7 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) client := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}, &v1alpha1.History{}). + WithStatusSubresource(&v1alpha1.Decision{}). Build() controller := &FilterWeigherPipelineController{ @@ -395,7 +385,6 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) Client: client, Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), - HistoryManager: lib.HistoryClient{Client: client}, }, Monitor: lib.FilterWeigherPipelineMonitor{}, } @@ -411,7 +400,7 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) if tt.decision.Spec.ManilaRaw != nil { req := manilaRequest - req.Options = scheduling.Options{SkipHistory: !tt.createHistory} + req.Options = scheduling.Options{} raw, marshalErr := json.Marshal(req) if marshalErr != nil { t.Fatalf("Failed to marshal request with options: %v", marshalErr) @@ -424,23 +413,7 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) t.Errorf("Expected no error but got: %v", err) } - // Check if history CRD was created when expected - if tt.expectHistoryCreated { - var histories v1alpha1.HistoryList - deadline := time.Now().Add(2 * time.Second) - for { - if err := client.List(context.Background(), &histories); err != nil { - t.Fatalf("Failed to list histories: %v", err) - } - if len(histories.Items) > 0 { - break - } - if time.Now().After(deadline) { - t.Fatal("timed out waiting for history CRD to be created") - } - time.Sleep(5 * time.Millisecond) - } - } else if !tt.expectError { + if !tt.expectError { if tt.expectResult && tt.decision.Status.Result == nil { t.Error("expected decision result to be set in original decision object") } diff --git a/internal/scheduling/manila/history_cleanup.go b/internal/scheduling/manila/history_cleanup.go deleted file mode 100644 index 824755261..000000000 --- a/internal/scheduling/manila/history_cleanup.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package manila - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log/slog" - "net/http" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - - "github.com/cobaltcore-dev/cortex/pkg/keystone" - "github.com/cobaltcore-dev/cortex/pkg/sso" - "github.com/gophercloud/gophercloud/v2" - "github.com/gophercloud/gophercloud/v2/openstack" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type HistoryCleanupConfig struct { - // Secret ref to keystone credentials stored in a k8s secret. - KeystoneSecretRef corev1.SecretReference `json:"keystoneSecretRef"` - // Secret ref to SSO credentials stored in a k8s secret, if applicable. - SSOSecretRef *corev1.SecretReference `json:"ssoSecretRef"` -} - -// Delete all history entries for manila shares that have been deleted. -func HistoryCleanup(ctx context.Context, client client.Client, conf HistoryCleanupConfig) error { - var authenticatedHTTP = http.DefaultClient - if conf.SSOSecretRef != nil { - var err error - authenticatedHTTP, err = sso.Connector{Client: client}. - FromSecretRef(ctx, *conf.SSOSecretRef) - if err != nil { - return err - } - } - authenticatedKeystone, err := keystone.Connector{Client: client, HTTPClient: authenticatedHTTP}. - FromSecretRef(ctx, conf.KeystoneSecretRef) - if err != nil { - return err - } - pc := authenticatedKeystone.Client() - // Workaround to find the v2 service of manila. - // See: https://github.com/gophercloud/gophercloud/issues/3347 - gophercloud.ServiceTypeAliases["shared-file-system"] = []string{"sharev2"} - manilaSC, err := openstack.NewSharedFileSystemV2(pc, gophercloud.EndpointOpts{ - Type: "sharev2", - Availability: gophercloud.Availability(authenticatedKeystone.Availability()), - }) - if err != nil { - return err - } - manilaSC.Microversion = "2.65" - - initialURL := manilaSC.Endpoint + "shares/detail?all_tenants=true" - var nextURL = &initialURL - var shares []struct { - ID string `json:"id"` - } - - for nextURL != nil { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, *nextURL, http.NoBody) - if err != nil { - return err - } - req.Header.Set("X-Auth-Token", manilaSC.Token()) - req.Header.Set("X-OpenStack-Manila-API-Version", manilaSC.Microversion) - resp, err := manilaSC.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - var list struct { - Shares []struct { - ID string `json:"id"` - } `json:"shares"` - Links []struct { - Rel string `json:"rel"` - Href string `json:"href"` - } `json:"shares_links"` - } - err = json.NewDecoder(resp.Body).Decode(&list) - if err != nil { - return err - } - shares = append(shares, list.Shares...) - nextURL = nil - for _, link := range list.Links { - if link.Rel == "next" { - nextURL = &link.Href - break - } - } - } - - if len(shares) == 0 { - return errors.New("no shares found") - } - slog.Info("found shares", "count", len(shares)) - sharesByID := make(map[string]struct{}) - for _, share := range shares { - sharesByID[share.ID] = struct{}{} - } - - // List all history and delete those whose share no longer exists. - historyList := &v1alpha1.HistoryList{} - if err := client.List(ctx, historyList); err != nil { - return err - } - for _, history := range historyList.Items { - // Skip non-manila histories. - if history.Spec.SchedulingDomain != v1alpha1.SchedulingDomainManila { - continue - } - // Skip histories for which the share still exists. - if _, ok := sharesByID[history.Spec.ResourceID]; ok { - continue - } - // Delete the history entry since the share no longer exists. - slog.Info("deleting history entry for deleted share", "history", history.Name, "shareID", history.Spec.ResourceID) - if err := client.Delete(ctx, &history); err != nil { - return err - } - } - return nil -} diff --git a/internal/scheduling/manila/history_cleanup_test.go b/internal/scheduling/manila/history_cleanup_test.go deleted file mode 100644 index f2ea5f914..000000000 --- a/internal/scheduling/manila/history_cleanup_test.go +++ /dev/null @@ -1,438 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package manila - -import ( - "context" - "encoding/json" - "errors" - "net/http" - "net/http/httptest" - "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - - corev1 "k8s.io/api/core/v1" -) - -func TestCleanupManila(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add corev1 scheme: %v", err) - } - - tests := []struct { - name string - histories []v1alpha1.History - expectError bool - authError bool - endpointError bool - mockServerError bool - emptySharesError bool - mockShares []mockShare - expectedDeleted []string - }{ - { - name: "handle authentication error", - histories: []v1alpha1.History{}, - authError: true, - expectError: true, - }, - { - name: "handle endpoint error", - histories: []v1alpha1.History{}, - endpointError: true, - expectError: true, - }, - { - name: "handle server error", - histories: []v1alpha1.History{}, - mockServerError: true, - expectError: true, - }, - { - name: "handle empty shares case", - histories: []v1alpha1.History{}, - emptySharesError: true, - expectError: true, - }, - { - name: "delete history for non-existent shares", - histories: []v1alpha1.History{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-existing-share", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - ResourceID: "share-exists", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-deleted-share", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - ResourceID: "share-deleted", - }, - }, - }, - mockShares: []mockShare{ - {ID: "share-exists"}, - }, - expectedDeleted: []string{"history-deleted-share"}, - expectError: false, - }, - { - name: "keep history for existing shares", - histories: []v1alpha1.History{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-share-1", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - ResourceID: "share-1", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-share-2", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainManila, - ResourceID: "share-2", - }, - }, - }, - mockShares: []mockShare{ - {ID: "share-1"}, - {ID: "share-2"}, - }, - expectedDeleted: []string{}, - expectError: false, - }, - { - name: "skip non-manila histories", - histories: []v1alpha1.History{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-other-type", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - ResourceID: "some-resource", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-other-operator", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: "other-operator", - ResourceID: "share-1", - }, - }, - }, - mockShares: []mockShare{{ID: "dummy-share"}}, // Add at least one share to avoid "no shares found" error - expectedDeleted: []string{}, - expectError: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := make([]client.Object, len(tt.histories)) - for i := range tt.histories { - objects[i] = &tt.histories[i] - } - - // Create mock Manila server - manilaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if tt.mockServerError { - w.WriteHeader(http.StatusInternalServerError) - return - } - - // Handle shares list endpoint - if r.URL.Path == "/shares" || r.URL.Path == "/shares/detail" { - w.Header().Set("Content-Type", "application/json") - if tt.emptySharesError { - // Return empty shares list - sharesResponse := map[string]any{ - "shares": []mockShare{}, - } - err := json.NewEncoder(w).Encode(sharesResponse) - if err != nil { - t.Errorf("Failed to encode shares response: %v", err) - } - return - } - - sharesResponse := map[string]any{ - "shares": tt.mockShares, - } - err := json.NewEncoder(w).Encode(sharesResponse) - if err != nil { - t.Errorf("Failed to encode shares response: %v", err) - } - return - } - - // Handle root path for service discovery - if r.URL.Path == "/" { - w.Header().Set("Content-Type", "application/json") - versionResponse := map[string]any{ - "versions": []map[string]any{ - { - "status": "CURRENT", - "id": "v2.0", - "links": []map[string]any{ - { - "href": "http://" + r.Host + "/v2/", - "rel": "self", - }, - }, - }, - }, - } - err := json.NewEncoder(w).Encode(versionResponse) - if err != nil { - t.Errorf("Failed to encode version response: %v", err) - } - return - } - - // Default response for other endpoints - w.WriteHeader(http.StatusNotFound) - })) - defer manilaServer.Close() - - // Create mock Keystone server - keystoneServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if tt.authError { - w.WriteHeader(http.StatusUnauthorized) - return - } - - w.Header().Set("Content-Type", "application/json") - - // Handle different Keystone API endpoints - switch r.URL.Path { - case "/", "/v3", "/v3/": - // Handle version discovery - versionResponse := map[string]any{ - "versions": map[string]any{ - "values": []map[string]any{ - { - "status": "stable", - "id": "v3.0", - "links": []map[string]any{ - { - "href": "http://" + r.Host + "/v3/", - "rel": "self", - }, - }, - }, - }, - }, - } - err := json.NewEncoder(w).Encode(versionResponse) - if err != nil { - t.Errorf("Failed to encode version response: %v", err) - } - case "/v3/auth/tokens": - w.WriteHeader(http.StatusCreated) - - // Mock token response - tokenResponse := map[string]any{ - "token": map[string]any{ - "methods": []string{"password"}, - "expires_at": "2030-01-01T00:00:00Z", - "project": map[string]any{ - "domain": map[string]any{ - "id": "default", - "name": "default", - }, - "id": "test-project-id", - "name": "test-project", - }, - "catalog": []map[string]any{ - { - "type": "sharev2", - "id": "manila-service-id", - "name": "manilav2", - "endpoints": []map[string]any{ - { - "region_id": "RegionOne", - "url": manilaServer.URL, - "region": "RegionOne", - "interface": "public", - "id": "manila-endpoint-id", - }, - }, - }, - }, - "user": map[string]any{ - "domain": map[string]any{ - "id": "default", - "name": "default", - }, - "id": "test-user-id", - "name": "test-user", - }, - }, - } - if tt.endpointError { - // Don't include sharev2 service in catalog - tokenResponse["token"].(map[string]any)["catalog"] = []map[string]any{} - } - - // Set the token in the header - w.Header().Set("X-Subject-Token", "mock-token-id") - err := json.NewEncoder(w).Encode(tokenResponse) - if err != nil { - t.Errorf("Failed to encode token response: %v", err) - } - default: - w.WriteHeader(http.StatusNotFound) - } - })) - defer keystoneServer.Close() - - // Add the keystone secret object - objects = append(objects, &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "keystone-secret", - Namespace: "default", - }, - Data: map[string][]byte{ - "url": []byte(keystoneServer.URL), - "availability": []byte("public"), - "username": []byte("test-user"), - "password": []byte("test-password"), - "projectName": []byte("test-project"), - "userDomainName": []byte("default"), - "projectDomainName": []byte("default"), - }, - }) - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - Build() - config := HistoryCleanupConfig{ - KeystoneSecretRef: corev1.SecretReference{ - Name: "keystone-secret", - Namespace: "default", - }, - } - err := HistoryCleanup(context.Background(), client, config) - - if tt.expectError && err == nil { - t.Error("Expected error but got none") - } - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - - if !tt.expectError { - // Verify expected history entries were deleted - for _, expectedDeleted := range tt.expectedDeleted { - var history v1alpha1.History - err := client.Get(context.Background(), - types.NamespacedName{Name: expectedDeleted}, &history) - if err == nil { - t.Errorf("Expected history %s to be deleted but it still exists", expectedDeleted) - } - } - - // Verify other histories still exist - for _, originalHistory := range tt.histories { - shouldBeDeleted := false - for _, expectedDeleted := range tt.expectedDeleted { - if originalHistory.Name == expectedDeleted { - shouldBeDeleted = true - break - } - } - if !shouldBeDeleted { - var history v1alpha1.History - err := client.Get(context.Background(), - types.NamespacedName{Name: originalHistory.Name}, &history) - if err != nil { - t.Errorf("Expected history %s to still exist but got error: %v", - originalHistory.Name, err) - } - } - } - } - }) - } -} - -func TestCleanupManilaHistoryCancel(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add corev1 scheme: %v", err) - } - - objects := []client.Object{ - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "keystone-secret", - Namespace: "default", - }, - Data: map[string][]byte{ - "url": []byte("http://invalid-keystone-url"), - "availability": []byte("public"), - "username": []byte("test-user"), - "password": []byte("test-password"), - "projectName": []byte("test-project"), - "userDomainName": []byte("default"), - "projectDomainName": []byte("default"), - }, - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - Build() - - config := HistoryCleanupConfig{ - KeystoneSecretRef: corev1.SecretReference{ - Name: "keystone-secret", - Namespace: "default", - }, - } - - ctx, cancel := context.WithTimeout(context.Background(), 0) - defer cancel() - - // This should exit quickly due to context cancellation - if err := HistoryCleanup(ctx, client, config); err != nil { - if !errors.Is(err, context.DeadlineExceeded) { - t.Errorf("Unexpected error during cleanup: %v", err) - } - } - - // If we reach here without hanging, the test passed -} - -type mockShare struct { - ID string `json:"id"` -} diff --git a/internal/scheduling/nova/cr_metrics_test.go b/internal/scheduling/nova/cr_metrics_test.go index 2aa7a2cce..6b0ac83c3 100644 --- a/internal/scheduling/nova/cr_metrics_test.go +++ b/internal/scheduling/nova/cr_metrics_test.go @@ -359,7 +359,7 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(flavorKnowledge). - WithStatusSubresource(&v1alpha1.Decision{}, &v1alpha1.History{}). + WithStatusSubresource(&v1alpha1.Decision{}). Build() counter := crs.NewNoHostFoundCounter() @@ -371,7 +371,6 @@ func TestFeatureGate_CommittedResourceTracking(t *testing.T) { Client: fakeClient, Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), - HistoryManager: lib.HistoryClient{Client: fakeClient}, }, FeatureGates: FeatureGates{CommittedResourceTracking: enabled}, CRRecorder: crs.Recorder{ diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller.go b/internal/scheduling/nova/filter_weigher_pipeline_controller.go index 451298928..22553464d 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller.go @@ -140,27 +140,6 @@ func (c *FilterWeigherPipelineController) ProcessNewDecisionFromAPI(ctx context. return err } -func (c *FilterWeigherPipelineController) upsertHistory(ctx context.Context, decision *v1alpha1.Decision, pipelineErr error) { - log := ctrl.LoggerFrom(ctx) - - var az *string - - if decision.Spec.NovaRaw != nil { - var request api.ExternalSchedulerRequest - err := json.Unmarshal(decision.Spec.NovaRaw.Raw, &request) - if err != nil { - log.Error(err, "failed to unmarshal novaRaw for history, using defaults") - } else { - azStr := request.Spec.Data.AvailabilityZone - az = &azStr - } - } - - if upsertErr := c.HistoryManager.CreateOrUpdateHistory(ctx, decision, az, pipelineErr); upsertErr != nil { - log.Error(upsertErr, "failed to create/update history") - } -} - func (c *FilterWeigherPipelineController) process(ctx context.Context, decision *v1alpha1.Decision) (*api.ExternalSchedulerRequest, error) { log := ctrl.LoggerFrom(ctx) startedAt := time.Now() // So we can measure sync duration. @@ -197,16 +176,13 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision } result, err := pipeline.Run(request) - // Persist the result on the decision before upserting history so that - // CreateOrUpdateHistory can observe the target host and ordered hosts. - // On error the result is empty/meaningless, so only set it on success. if err == nil { decision.Status.Result = &result } - if !request.Options.SkipHistory { - c.upsertHistory(ctx, decision, err) - } - + // Record a human-readable explanation of the decision so it propagates to + // the VM CortexSpec.Explanation. On error the result is nil and the + // explanation carries the failure reason. + decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) // We create a virtual machine resource based on the scheduling decision, // logging the specific reason whenever we skip creation so it can be traced. instanceUUID := request.Spec.Data.InstanceUUID @@ -276,7 +252,6 @@ func (c *FilterWeigherPipelineController) InitPipeline( func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, mcl *multicluster.Client) error { c.Initializer = c c.SchedulingDomain = v1alpha1.SchedulingDomainNova - c.HistoryManager = lib.HistoryClient{Client: mcl, Recorder: mcl.GetEventRecorder("cortex-nova-scheduler")} c.gatherer = &candidateGatherer{Client: mcl} c.vmHandler = &VirtualMachineHandler{Client: mcl} if err := mgr.Add(manager.RunnableFunc(c.InitAllPipelines)); err != nil { diff --git a/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go b/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go index ccd565f56..f7c4f5a41 100644 --- a/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/nova/filter_weigher_pipeline_controller_test.go @@ -9,10 +9,8 @@ import ( "errors" "strings" "testing" - "time" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -78,7 +76,7 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { }, Weights: map[string]float64{"compute-1": 1.0, "compute-2": 0.5}, Pipeline: "test-pipeline", - Options: scheduling.Options{SkipHistory: true}, + Options: scheduling.Options{}, } novaRaw, err := json.Marshal(novaRequest) @@ -401,15 +399,10 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) pipeline *v1alpha1.Pipeline pipelineConf *v1alpha1.Pipeline setupPipelineConfigs bool - createHistory bool expectError bool expectResult bool - expectHistoryCreated bool expectUpdatedStatus bool errorContains string - // verifyHistory, if set, is invoked with the persisted History CRD so - // tests can assert the decision was written through correctly. - verifyHistory func(t *testing.T, history *v1alpha1.History) }{ { name: "successful processing with decision creation enabled", @@ -452,41 +445,9 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) }, }, setupPipelineConfigs: true, - createHistory: true, expectError: false, expectResult: true, - expectHistoryCreated: true, expectUpdatedStatus: true, - // Regression guard: the History CRD must reflect the actual - // decision (target host selected, ordered hosts, Ready=True). - // Previously the result was written to the decision *after* the - // history upsert, so history always recorded a failed decision - // with no target host. - verifyHistory: func(t *testing.T, history *v1alpha1.History) { - cur := history.Status.Current - if !cur.Successful { - t.Errorf("expected history current.successful=true, got false") - } - if cur.TargetHost == nil { - t.Errorf("expected history current.targetHost to be set, got nil") - } - if len(cur.OrderedHosts) == 0 { - t.Errorf("expected history current.orderedHosts to be populated, got empty") - } - if cur.Explanation == "" { - t.Errorf("expected history current.explanation to be set, got empty") - } - ready := meta.FindStatusCondition(history.Status.Conditions, v1alpha1.HistoryConditionReady) - if ready == nil { - t.Fatalf("expected Ready condition to be set") - } - if ready.Status != metav1.ConditionTrue { - t.Errorf("expected Ready condition status True, got %s", ready.Status) - } - if ready.Reason != v1alpha1.HistoryReasonSchedulingSucceeded { - t.Errorf("expected Ready reason %q, got %q", v1alpha1.HistoryReasonSchedulingSucceeded, ready.Reason) - } - }, }, { name: "successful processing with decision creation disabled", @@ -529,10 +490,8 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) }, }, setupPipelineConfigs: true, - createHistory: false, expectError: false, expectResult: true, - expectHistoryCreated: false, expectUpdatedStatus: false, }, { @@ -557,7 +516,6 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) setupPipelineConfigs: false, expectError: true, expectResult: false, - expectHistoryCreated: false, expectUpdatedStatus: false, errorContains: "pipeline not found or not ready", }, @@ -599,10 +557,8 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) }, }, setupPipelineConfigs: true, - createHistory: true, expectError: true, expectResult: false, - expectHistoryCreated: false, expectUpdatedStatus: false, errorContains: "no novaRaw spec defined", }, @@ -636,10 +592,8 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) }, }, setupPipelineConfigs: true, - createHistory: true, expectError: true, expectResult: false, - expectHistoryCreated: false, expectUpdatedStatus: false, errorContains: "pipeline not found or not ready", }, @@ -673,10 +627,8 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) }, }, setupPipelineConfigs: true, - createHistory: true, expectError: true, expectResult: false, - expectHistoryCreated: false, expectUpdatedStatus: false, errorContains: "pipeline not found or not ready", }, @@ -692,7 +644,7 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) client := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}, &v1alpha1.History{}). + WithStatusSubresource(&v1alpha1.Decision{}). Build() controller := &FilterWeigherPipelineController{ @@ -700,7 +652,6 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) Client: client, Pipelines: make(map[string]lib.FilterWeigherPipeline[api.ExternalSchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), - HistoryManager: lib.HistoryClient{Client: client}, }, Monitor: lib.FilterWeigherPipelineMonitor{}, } @@ -726,7 +677,7 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) if tt.decision.Spec.NovaRaw != nil { req := novaRequest - req.Options = scheduling.Options{SkipHistory: !tt.createHistory} + req.Options = scheduling.Options{} raw, marshalErr := json.Marshal(req) if marshalErr != nil { t.Fatalf("Failed to marshal request with options: %v", marshalErr) @@ -748,35 +699,6 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T) t.Errorf("Expected error to contain %q, got: %v", tt.errorContains, err) } - // Check if history CRD was created when expected - if tt.expectHistoryCreated { - var histories v1alpha1.HistoryList - deadline := time.Now().Add(2 * time.Second) - for { - if err := client.List(context.Background(), &histories); err != nil { - t.Fatalf("Failed to list histories: %v", err) - } - if len(histories.Items) > 0 { - break - } - if time.Now().After(deadline) { - t.Fatal("timed out waiting for history CRD to be created") - } - time.Sleep(5 * time.Millisecond) - } - if tt.verifyHistory != nil { - tt.verifyHistory(t, &histories.Items[0]) - } - } else { - var histories v1alpha1.HistoryList - if err := client.List(context.Background(), &histories); err != nil { - t.Fatalf("Failed to list histories: %v", err) - } - if len(histories.Items) != 0 { - t.Error("Expected no history CRD but found one") - } - } - // Validate result expectations if tt.expectResult && tt.decision.Status.Result == nil { t.Error("Expected result to be set but was nil") @@ -819,7 +741,7 @@ func TestFilterWeigherPipelineController_IgnorePreselection(t *testing.T) { }, Weights: map[string]float64{"original-host-1": 1.0, "original-host-2": 0.5}, Pipeline: "test-pipeline", - Options: scheduling.Options{SkipHistory: true}, + Options: scheduling.Options{}, } novaRaw, err := json.Marshal(novaRequest) diff --git a/internal/scheduling/nova/history_cleanup.go b/internal/scheduling/nova/history_cleanup.go deleted file mode 100644 index 72f4d5583..000000000 --- a/internal/scheduling/nova/history_cleanup.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package nova - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - "net/http" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - corev1 "k8s.io/api/core/v1" - - "github.com/cobaltcore-dev/cortex/pkg/keystone" - "github.com/cobaltcore-dev/cortex/pkg/sso" - "github.com/gophercloud/gophercloud/v2" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type HistoryCleanupConfig struct { - // Secret ref to keystone credentials stored in a k8s secret. - KeystoneSecretRef corev1.SecretReference `json:"keystoneSecretRef"` - // Secret ref to SSO credentials stored in a k8s secret, if applicable. - SSOSecretRef *corev1.SecretReference `json:"ssoSecretRef"` -} - -// Delete all histories for nova servers that have been deleted. -func HistoryCleanup(ctx context.Context, client client.Client, conf HistoryCleanupConfig) error { - var authenticatedHTTP = http.DefaultClient - if conf.SSOSecretRef != nil { - var err error - authenticatedHTTP, err = sso.Connector{Client: client}. - FromSecretRef(ctx, *conf.SSOSecretRef) - if err != nil { - return err - } - } - authenticatedKeystone, err := keystone. - Connector{Client: client, HTTPClient: authenticatedHTTP}. - FromSecretRef(ctx, conf.KeystoneSecretRef) - if err != nil { - return err - } - pc := authenticatedKeystone.Client() - novaURL, err := pc.EndpointLocator(gophercloud.EndpointOpts{ - Type: "compute", - Availability: gophercloud.Availability(authenticatedKeystone.Availability()), - }) - if err != nil { - return err - } - novaSC := &gophercloud.ServiceClient{ - ProviderClient: pc, - Endpoint: novaURL, - // Since 2.53, the hypervisor id and service id is a UUID. - // Since 2.61, the extra_specs are returned in the flavor details. - Microversion: "2.61", - } - - initialURL := novaSC.Endpoint + "servers/detail?all_tenants=true" - var nextURL = &initialURL - var servers []struct { - ID string `json:"id"` - } - - for nextURL != nil { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, *nextURL, http.NoBody) - if err != nil { - return err - } - req.Header.Set("X-Auth-Token", novaSC.Token()) - req.Header.Set("X-OpenStack-Nova-API-Version", novaSC.Microversion) - resp, err := novaSC.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - var list struct { - Servers []struct { - ID string `json:"id"` - } `json:"servers"` - Links []struct { - Rel string `json:"rel"` - Href string `json:"href"` - } `json:"servers_links"` - } - err = json.NewDecoder(resp.Body).Decode(&list) - if err != nil { - return err - } - servers = append(servers, list.Servers...) - nextURL = nil - for _, link := range list.Links { - if link.Rel == "next" { - nextURL = &link.Href - break - } - } - } - - slog.Info("found servers", "count", len(servers)) - serversByID := make(map[string]struct{}) - for _, server := range servers { - serversByID[server.ID] = struct{}{} - } - - // List all reservations. - reservationList := &v1alpha1.ReservationList{} - if err := client.List(ctx, reservationList); err != nil { - return err - } - reservationsByName := make(map[string]v1alpha1.Reservation) - for _, reservation := range reservationList.Items { - reservationsByName[reservation.Name] = reservation - } - - // List all histories and check if the server still exists. - historyList := &v1alpha1.HistoryList{} - if err := client.List(ctx, historyList); err != nil { - return err - } - for _, history := range historyList.Items { - // Skip non-nova histories. - if history.Spec.SchedulingDomain != v1alpha1.SchedulingDomainNova { - continue - } - // Skip histories that are linked to existing reservations. - if _, ok := reservationsByName[history.Spec.ResourceID]; ok { - continue - } - // Skip histories for which the server still exists. - if _, ok := serversByID[history.Spec.ResourceID]; ok { - continue - } - // Delete the history since the server no longer exists. - slog.Info("deleting history for deleted server", "history", history.Name, "serverID", history.Spec.ResourceID) - if err := client.Delete(ctx, &history); err != nil { - return err - } - } - return nil -} diff --git a/internal/scheduling/nova/history_cleanup_test.go b/internal/scheduling/nova/history_cleanup_test.go deleted file mode 100644 index 3123a8368..000000000 --- a/internal/scheduling/nova/history_cleanup_test.go +++ /dev/null @@ -1,440 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package nova - -import ( - "context" - "encoding/json" - "errors" - "net/http" - "net/http/httptest" - "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - - corev1 "k8s.io/api/core/v1" -) - -func TestCleanupNova(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add v1alpha1 scheme: %v", err) - } - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add corev1 scheme: %v", err) - } - - tests := []struct { - name string - histories []v1alpha1.History - reservations []v1alpha1.Reservation - mockServers []mockServer - authError bool - endpointError bool - serverError bool - emptyServers bool - expectedDeleted []string - expectError bool - }{ - { - name: "authentication error", - histories: []v1alpha1.History{}, - authError: true, - expectError: true, - }, - { - name: "endpoint discovery error", - histories: []v1alpha1.History{}, - endpointError: true, - expectError: true, - }, - { - name: "nova server error", - histories: []v1alpha1.History{}, - serverError: true, - expectError: true, - }, - { - name: "no servers found", - histories: []v1alpha1.History{}, - emptyServers: true, - expectError: false, - }, - { - name: "delete histories for non-existent servers", - histories: []v1alpha1.History{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-existing-server", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "server-exists", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-deleted-server", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "server-deleted", - }, - }, - }, - mockServers: []mockServer{ - {ID: "server-exists"}, - }, - expectedDeleted: []string{"history-deleted-server"}, - expectError: false, - }, - { - name: "keep histories for existing servers", - histories: []v1alpha1.History{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-server-1", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "server-1", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-server-2", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "server-2", - }, - }, - }, - mockServers: []mockServer{ - {ID: "server-1"}, - {ID: "server-2"}, - }, - expectedDeleted: []string{}, - expectError: false, - }, - { - name: "skip histories with linked reservations", - histories: []v1alpha1.History{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-reserved-server", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "server-reserved", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-unreserved-server", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainNova, - ResourceID: "server-unreserved", - }, - }, - }, - reservations: []v1alpha1.Reservation{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "server-reserved", - }, - }, - }, - mockServers: []mockServer{}, - expectedDeleted: []string{"history-unreserved-server"}, - expectError: false, - }, - { - name: "skip non-nova histories", - histories: []v1alpha1.History{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-cinder", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: v1alpha1.SchedulingDomainCinder, - ResourceID: "volume-1", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "history-wrong-operator", - }, - Spec: v1alpha1.HistorySpec{ - SchedulingDomain: "other-operator", - ResourceID: "server-1", - }, - }, - }, - mockServers: []mockServer{}, - expectedDeleted: []string{}, - expectError: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - objects := make([]client.Object, 0, len(tt.histories)+len(tt.reservations)) - for i := range tt.histories { - objects = append(objects, &tt.histories[i]) - } - for i := range tt.reservations { - objects = append(objects, &tt.reservations[i]) - } - - novaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if tt.serverError { - w.WriteHeader(http.StatusInternalServerError) - return - } - - if r.URL.Path == "/servers" || r.URL.Path == "/servers/detail" { - w.Header().Set("Content-Type", "application/json") - if tt.emptyServers { - serversResponse := map[string]any{ - "servers": []mockServer{}, - } - err := json.NewEncoder(w).Encode(serversResponse) - if err != nil { - t.Errorf("Failed to encode servers response: %v", err) - } - return - } - - serversResponse := map[string]any{ - "servers": tt.mockServers, - } - err := json.NewEncoder(w).Encode(serversResponse) - if err != nil { - t.Errorf("Failed to encode servers response: %v", err) - } - return - } - - w.WriteHeader(http.StatusNotFound) - })) - defer novaServer.Close() - - keystoneServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if tt.authError { - w.WriteHeader(http.StatusUnauthorized) - return - } - - w.Header().Set("Content-Type", "application/json") - - switch r.URL.Path { - case "/", "/v3", "/v3/": - versionResponse := map[string]any{ - "versions": map[string]any{ - "values": []map[string]any{ - { - "status": "stable", - "id": "v3.0", - "links": []map[string]any{ - { - "href": "http://" + r.Host + "/v3/", - "rel": "self", - }, - }, - }, - }, - }, - } - err := json.NewEncoder(w).Encode(versionResponse) - if err != nil { - t.Errorf("Failed to encode version response: %v", err) - } - case "/v3/auth/tokens": - w.WriteHeader(http.StatusCreated) - - tokenResponse := map[string]any{ - "token": map[string]any{ - "methods": []string{"password"}, - "expires_at": "2030-01-01T00:00:00Z", - "project": map[string]any{ - "domain": map[string]any{ - "id": "default", - "name": "default", - }, - "id": "test-project-id", - "name": "test-project", - }, - "catalog": []map[string]any{ - { - "type": "compute", - "id": "nova-service-id", - "name": "nova", - "endpoints": []map[string]any{ - { - "region_id": "RegionOne", - "url": novaServer.URL, - "region": "RegionOne", - "interface": "public", - "id": "nova-endpoint-id", - }, - }, - }, - }, - "user": map[string]any{ - "domain": map[string]any{ - "id": "default", - "name": "default", - }, - "id": "test-user-id", - "name": "test-user", - }, - }, - } - if tt.endpointError { - tokenResponse["token"].(map[string]any)["catalog"] = []map[string]any{} - } - - w.Header().Set("X-Subject-Token", "mock-token-id") - err := json.NewEncoder(w).Encode(tokenResponse) - if err != nil { - t.Errorf("Failed to encode token response: %v", err) - } - default: - w.WriteHeader(http.StatusNotFound) - } - })) - defer keystoneServer.Close() - - // Add the keystone secret object - objects = append(objects, &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "keystone-secret", - Namespace: "default", - }, - Data: map[string][]byte{ - "url": []byte(keystoneServer.URL), - "availability": []byte("public"), - "username": []byte("test-user"), - "password": []byte("test-password"), - "projectName": []byte("test-project"), - "userDomainName": []byte("default"), - "projectDomainName": []byte("default"), - }, - }) - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - Build() - config := HistoryCleanupConfig{ - KeystoneSecretRef: corev1.SecretReference{ - Name: "keystone-secret", - Namespace: "default", - }, - } - err := HistoryCleanup(context.Background(), client, config) - - if tt.expectError && err == nil { - t.Error("Expected error but got none") - } - if !tt.expectError && err != nil { - t.Errorf("Expected no error but got: %v", err) - } - - if !tt.expectError { - // Verify expected histories were deleted - for _, expectedDeleted := range tt.expectedDeleted { - var history v1alpha1.History - err := client.Get(context.Background(), - types.NamespacedName{Name: expectedDeleted}, &history) - if err == nil { - t.Errorf("Expected history %s to be deleted but it still exists", expectedDeleted) - } - } - - // Verify other histories still exist - for _, originalHistory := range tt.histories { - shouldBeDeleted := false - for _, expectedDeleted := range tt.expectedDeleted { - if originalHistory.Name == expectedDeleted { - shouldBeDeleted = true - break - } - } - if !shouldBeDeleted { - var history v1alpha1.History - err := client.Get(context.Background(), - types.NamespacedName{Name: originalHistory.Name}, &history) - if err != nil { - t.Errorf("Expected history %s to still exist but got error: %v", - originalHistory.Name, err) - } - } - } - } - }) - } -} - -func TestCleanupNovaHistoriesCancel(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add corev1 scheme: %v", err) - } - objects := []client.Object{ - &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "keystone-secret", - Namespace: "default", - }, - Data: map[string][]byte{ - "url": []byte("http://invalid-keystone-url"), - "availability": []byte("public"), - "username": []byte("test-user"), - "password": []byte("test-password"), - "projectName": []byte("test-project"), - "userDomainName": []byte("default"), - "projectDomainName": []byte("default"), - }, - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - Build() - - config := HistoryCleanupConfig{ - KeystoneSecretRef: corev1.SecretReference{ - Name: "keystone-secret", - Namespace: "default", - }, - } - - ctx, cancel := context.WithTimeout(context.Background(), 0) - defer cancel() - - // This should exit quickly due to context cancellation - if err := HistoryCleanup(ctx, client, config); err != nil { - if !errors.Is(err, context.DeadlineExceeded) { - t.Errorf("Unexpected error during cleanup: %v", err) - } - } - - // If we reach here without hanging, the test passed -} - -type mockServer struct { - ID string `json:"id"` -} diff --git a/internal/scheduling/nova/integration_test.go b/internal/scheduling/nova/integration_test.go index 64a9bb35a..f22ff3523 100644 --- a/internal/scheduling/nova/integration_test.go +++ b/internal/scheduling/nova/integration_test.go @@ -252,7 +252,6 @@ func NewIntegrationTestServer(t *testing.T, pipelineConfig PipelineConfig, objec Client: k8sClient, Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), - HistoryManager: lib.HistoryClient{Client: k8sClient}, }, Monitor: getSharedMonitor(), } diff --git a/internal/scheduling/pods/filter_weigher_pipeline_controller.go b/internal/scheduling/pods/filter_weigher_pipeline_controller.go index cf27558ed..8619e4cb5 100644 --- a/internal/scheduling/pods/filter_weigher_pipeline_controller.go +++ b/internal/scheduling/pods/filter_weigher_pipeline_controller.go @@ -147,20 +147,17 @@ func (c *FilterWeigherPipelineController) process(ctx context.Context, decision return errors.New("no nodes available for scheduling") } - // Execute the scheduling pipeline. Options not set: pod scheduling always records history. + // Execute the scheduling pipeline. request := pods.PodPipelineRequest{Nodes: nodes.Items, Pod: *pod} result, err := pipeline.Run(request) - // Persist the result on the decision before upserting history so that - // CreateOrUpdateHistory can observe the target host and ordered hosts. - // On error the result is empty/meaningless, so only set it on success. + if err == nil { decision.Status.Result = &result } - if !request.Options.SkipHistory { - if upsertErr := c.HistoryManager.CreateOrUpdateHistory(ctx, decision, nil, err); upsertErr != nil { - log.Error(upsertErr, "failed to create/update history") - } - } + // Record a human-readable explanation of the decision for downstream + // consumers. On error the result is nil and the explanation carries the + // failure reason. + decision.Status.Explanation = lib.GenerateExplanation(decision.Status.Result, err) if err != nil { log.V(1).Error(err, "failed to run scheduler pipeline") return errors.New("failed to run scheduler pipeline") @@ -220,22 +217,12 @@ func (c *FilterWeigherPipelineController) handlePod() handler.EventHandler { log.Error(err, "failed to process new pod for scheduling", "pod", newPod.Name) } }, - DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { - c.processMu.Lock() - defer c.processMu.Unlock() - pod := evt.Object.(*corev1.Pod) - if err := c.HistoryManager.Delete(ctx, v1alpha1.SchedulingDomainPods, pod.Namespace+"--"+pod.Name); err != nil { - log := ctrl.LoggerFrom(ctx) - log.Error(err, "failed to delete history CRD for pod", "pod", pod.Name) - } - }, } } func (c *FilterWeigherPipelineController) SetupWithManager(mgr manager.Manager, mcl *multicluster.Client) error { c.Initializer = c c.SchedulingDomain = v1alpha1.SchedulingDomainPods - c.HistoryManager = lib.HistoryClient{Client: mcl, Recorder: mcl.GetEventRecorder("cortex-pods-scheduler")} if err := mgr.Add(manager.RunnableFunc(c.InitAllPipelines)); err != nil { return err } diff --git a/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go b/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go index 7a8159be6..0c9ad6fb8 100644 --- a/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go +++ b/internal/scheduling/pods/filter_weigher_pipeline_controller_test.go @@ -6,7 +6,6 @@ package pods import ( "context" "testing" - "time" "github.com/cobaltcore-dev/cortex/api/external/pods" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -122,7 +121,6 @@ func TestFilterWeigherPipelineController_Reconcile(t *testing.T) { Pipelines: map[string]lib.FilterWeigherPipeline[pods.PodPipelineRequest]{ "pods-scheduler": createMockPodPipeline(), }, - HistoryManager: lib.HistoryClient{Client: client}, }, Monitor: lib.FilterWeigherPipelineMonitor{}, } @@ -265,15 +263,13 @@ func TestFilterWeigherPipelineController_ProcessNewPod(t *testing.T) { } tests := []struct { - name string - pod *corev1.Pod - nodes []corev1.Node - pipelineConfig *v1alpha1.Pipeline - createHistory bool - expectError bool - expectHistoryCreated bool - expectNodeAssigned bool - expectTargetHost string + name string + pod *corev1.Pod + nodes []corev1.Node + pipelineConfig *v1alpha1.Pipeline + expectError bool + expectNodeAssigned bool + expectTargetHost string }{ { name: "successful pod processing with decision creation", @@ -305,11 +301,9 @@ func TestFilterWeigherPipelineController_ProcessNewPod(t *testing.T) { Weighers: []v1alpha1.WeigherSpec{}, }, }, - createHistory: true, - expectError: false, - expectHistoryCreated: true, - expectNodeAssigned: true, - expectTargetHost: "node1", + expectError: false, + expectNodeAssigned: true, + expectTargetHost: "node1", }, { name: "successful pod processing without decision creation", @@ -338,11 +332,9 @@ func TestFilterWeigherPipelineController_ProcessNewPod(t *testing.T) { Weighers: []v1alpha1.WeigherSpec{}, }, }, - createHistory: false, - expectError: false, - expectHistoryCreated: true, - expectNodeAssigned: true, - expectTargetHost: "node1", + expectError: false, + expectNodeAssigned: true, + expectTargetHost: "node1", }, { name: "pipeline not configured", @@ -355,11 +347,10 @@ func TestFilterWeigherPipelineController_ProcessNewPod(t *testing.T) { SchedulerName: "", }, }, - nodes: []corev1.Node{}, - pipelineConfig: nil, - expectError: true, - expectHistoryCreated: false, - expectNodeAssigned: false, + nodes: []corev1.Node{}, + pipelineConfig: nil, + expectError: true, + expectNodeAssigned: false, }, { name: "no nodes available", @@ -384,10 +375,8 @@ func TestFilterWeigherPipelineController_ProcessNewPod(t *testing.T) { Weighers: []v1alpha1.WeigherSpec{}, }, }, - createHistory: true, - expectError: true, - expectHistoryCreated: false, - expectNodeAssigned: false, + expectError: true, + expectNodeAssigned: false, }, } @@ -404,14 +393,13 @@ func TestFilterWeigherPipelineController_ProcessNewPod(t *testing.T) { client := fake.NewClientBuilder(). WithScheme(scheme). WithRuntimeObjects(objects...). - WithStatusSubresource(&v1alpha1.Decision{}, &v1alpha1.History{}). + WithStatusSubresource(&v1alpha1.Decision{}). Build() controller := &FilterWeigherPipelineController{ BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[pods.PodPipelineRequest]]{ Pipelines: map[string]lib.FilterWeigherPipeline[pods.PodPipelineRequest]{}, PipelineConfigs: map[string]v1alpha1.Pipeline{}, - HistoryManager: lib.HistoryClient{Client: client}, }, Monitor: lib.FilterWeigherPipelineMonitor{}, } @@ -434,32 +422,6 @@ func TestFilterWeigherPipelineController_ProcessNewPod(t *testing.T) { return } - // Check if history CRD was created when expected - if tt.expectHistoryCreated { - var histories v1alpha1.HistoryList - deadline := time.Now().Add(2 * time.Second) - for { - if err := client.List(context.Background(), &histories); err != nil { - t.Fatalf("Failed to list histories: %v", err) - } - if len(histories.Items) > 0 { - break - } - if time.Now().After(deadline) { - t.Fatal("timed out waiting for history CRD to be created") - } - time.Sleep(5 * time.Millisecond) - } - } else { - var histories v1alpha1.HistoryList - if err := client.List(context.Background(), &histories); err != nil { - t.Fatalf("Failed to list histories: %v", err) - } - if len(histories.Items) != 0 { - t.Error("Expected no history CRD but found one") - } - } - // Check if node was assigned (if expected) if tt.expectNodeAssigned { var binding corev1.Binding diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index d32ff8e5b..caeab9556 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -829,7 +829,6 @@ func (c *Reconciler) probeScheduler( ReadOnly: true, AssumeEmptyHosts: ignoreAllocations, IgnoredReservationTypes: ignoredReservationTypes, - SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true, }) diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 8c19e1d71..ba10dfcd5 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -362,7 +362,6 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr AssumeEmptyHosts: false, IgnoredReservationTypes: nil, MaxCandidates: 1, - SkipHistory: true, SkipInflight: false, // TODO pessimistic blocking needed, will be addressed in follow up ticket SkipCommittedResourceTracking: true, // CR slot scheduling, not a VM placement } diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go index b7ad34d5f..fcbdaa7c2 100644 --- a/internal/scheduling/reservations/failover/integration_test.go +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -1146,7 +1146,6 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h Client: k8sClient, Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), - HistoryManager: lib.HistoryClient{Client: k8sClient}, }, Monitor: getSharedMonitor(), } @@ -1347,7 +1346,6 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, Client: k8sClient, Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), PipelineConfigs: make(map[string]v1alpha1.Pipeline), - HistoryManager: lib.HistoryClient{Client: k8sClient}, }, Monitor: getSharedMonitor(), } diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 7d7ea0c99..bc9d44a3b 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -19,14 +19,13 @@ import ( // Pipeline names for failover reservation scheduling const ( // PipelineNewFailoverReservation is used to find a host for creating a new reservation. - // Uses the general-purpose pipeline; LockReservations and SkipHistory are set via Options. + // Uses the general-purpose pipeline; LockReservations is set via Options. PipelineNewFailoverReservation = "kvm-general-purpose-load-balancing" ) // DefaultFailoverOptions is the base scheduling.Options for all failover scheduling calls. // Per-call overrides (ReadOnly, LockReservations) are applied on top. var DefaultFailoverOptions = scheduling.Options{ - SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true, } diff --git a/pkg/multicluster/recorder_test.go b/pkg/multicluster/recorder_test.go index 16318747c..c893ed014 100644 --- a/pkg/multicluster/recorder_test.go +++ b/pkg/multicluster/recorder_test.go @@ -70,19 +70,19 @@ func TestMultiClusterRecorder_HomeGVK(t *testing.T) { fakeRecorder: homeRecorder, } - historyGVK := schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "History"} + reservationGVK := schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "Reservation"} mcl := &Client{ HomeCluster: homeCluster, HomeScheme: scheme, - homeGVKs: map[schema.GroupVersionKind]bool{historyGVK: true}, + homeGVKs: map[schema.GroupVersionKind]bool{reservationGVK: true}, } recorder := mcl.GetEventRecorder("test-recorder") - history := &v1alpha1.History{ + reservation := &v1alpha1.Reservation{ ObjectMeta: metav1.ObjectMeta{Name: "nova-uuid-1"}, } - recorder.Eventf(history, nil, corev1.EventTypeNormal, "SchedulingSucceeded", "Scheduled", "selected host: %s", "compute-1") + recorder.Eventf(reservation, nil, corev1.EventTypeNormal, "SchedulingSucceeded", "Scheduled", "selected host: %s", "compute-1") calls := homeRecorder.getCalls() if len(calls) != 1 { @@ -241,11 +241,11 @@ func TestMultiClusterRecorder_ConcurrentEventf(t *testing.T) { homeRecorder := &fakeEventRecorder{} homeCluster := &fakeCluster{fakeRecorder: homeRecorder, fakeCache: &fakeCache{}} - historyGVK := schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "History"} + reservationGVK := schema.GroupVersionKind{Group: "cortex.cloud", Version: "v1alpha1", Kind: "Reservation"} mcl := &Client{ HomeCluster: homeCluster, HomeScheme: scheme, - homeGVKs: map[schema.GroupVersionKind]bool{historyGVK: true}, + homeGVKs: map[schema.GroupVersionKind]bool{reservationGVK: true}, } recorder := mcl.GetEventRecorder("test-recorder") @@ -256,10 +256,10 @@ func TestMultiClusterRecorder_ConcurrentEventf(t *testing.T) { for i := range goroutines { go func(n int) { defer wg.Done() - history := &v1alpha1.History{ - ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("history-%d", n)}, + reservation := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("reservation-%d", n)}, } - recorder.Eventf(history, nil, corev1.EventTypeNormal, "Test", "Test", "event %d", n) + recorder.Eventf(reservation, nil, corev1.EventTypeNormal, "Test", "Test", "event %d", n) }(i) } wg.Wait() diff --git a/pkg/multicluster/routers.go b/pkg/multicluster/routers.go index 44e6633b1..90c0abc99 100644 --- a/pkg/multicluster/routers.go +++ b/pkg/multicluster/routers.go @@ -18,7 +18,6 @@ import ( var DefaultResourceRouters = map[schema.GroupVersionKind]ResourceRouter{ {Group: "kvm.cloud.sap", Version: "v1", Kind: "Hypervisor"}: HypervisorResourceRouter{}, {Group: "cortex.cloud", Version: "v1alpha1", Kind: "Reservation"}: ReservationsResourceRouter{}, - {Group: "cortex.cloud", Version: "v1alpha1", Kind: "History"}: HistoryResourceRouter{}, {Group: "cortex.cloud", Version: "v1alpha1", Kind: "CommittedResource"}: CommittedResourceRouter{}, {Group: "cortex.cloud", Version: "v1alpha1", Kind: "ProjectQuota"}: ProjectQuotaResourceRouter{}, {Group: "cortex.cloud", Version: "v1alpha1", Kind: "FlavorGroupCapacity"}: FlavorGroupCapacityResourceRouter{}, @@ -225,53 +224,6 @@ func (f FlavorGroupCapacityResourceRouter) Match(obj any, labels map[string]stri return fgc.Spec.AvailabilityZone == availabilityZone, nil } -// HistoryResourceRouter routes histories to clusters based on availability zone. -type HistoryResourceRouter struct{} - -func (h HistoryResourceRouter) ExtractClusterSelector(obj any) (string, error) { - switch v := obj.(type) { - case *v1alpha1.History: - if v == nil { - return "", errors.New("object is nil") - } - if v.Spec.AvailabilityZone == nil || *v.Spec.AvailabilityZone == "" { - return "", errors.New("history does not have availability zone in spec") - } - return *v.Spec.AvailabilityZone, nil - case v1alpha1.History: - if v.Spec.AvailabilityZone == nil || *v.Spec.AvailabilityZone == "" { - return "", errors.New("history does not have availability zone in spec") - } - return *v.Spec.AvailabilityZone, nil - default: - return "", errors.New("object is not a History") - } -} - -func (h HistoryResourceRouter) Match(obj any, labels map[string]string) (bool, error) { - var hist v1alpha1.History - - switch v := obj.(type) { - case *v1alpha1.History: - if v == nil { - return false, errors.New("object is nil") - } - hist = *v - case v1alpha1.History: - hist = v - default: - return false, errors.New("object is not a History") - } - availabilityZone, ok := labels["availabilityZone"] - if !ok { - return false, errors.New("cluster does not have availabilityZone label") - } - if hist.Spec.AvailabilityZone == nil || *hist.Spec.AvailabilityZone == "" { - return false, errors.New("history does not have availability zone in spec") - } - return *hist.Spec.AvailabilityZone == availabilityZone, nil -} - // ProjectQuotaResourceRouter routes project quotas to clusters based on availability zone. type ProjectQuotaResourceRouter struct{} diff --git a/pkg/multicluster/routers_test.go b/pkg/multicluster/routers_test.go index 882bf1ba8..74af9abfa 100644 --- a/pkg/multicluster/routers_test.go +++ b/pkg/multicluster/routers_test.go @@ -108,115 +108,6 @@ func TestHypervisorResourceRouter_Match(t *testing.T) { } } -func TestHistoryResourceRouter_Match(t *testing.T) { - router := HistoryResourceRouter{} - - az := "qa-west" - azOther := "qa-east^" - empty := "" - - tests := []struct { - name string - obj any - labels map[string]string - wantMatch bool - wantErr bool - }{ - { - name: "matching AZ", - obj: v1alpha1.History{ - Spec: v1alpha1.HistorySpec{ - AvailabilityZone: &az, - }, - }, - labels: map[string]string{"availabilityZone": az}, - wantMatch: true, - }, - { - name: "matching AZ pointer", - obj: new(v1alpha1.History{ - Spec: v1alpha1.HistorySpec{ - AvailabilityZone: &az, - }, - }), - labels: map[string]string{"availabilityZone": az}, - wantMatch: true, - }, - { - name: "non-matching AZ", - obj: v1alpha1.History{ - Spec: v1alpha1.HistorySpec{ - AvailabilityZone: &az, - }, - }, - labels: map[string]string{"availabilityZone": azOther}, - wantMatch: false, - }, - { - name: "not a History", - obj: "not-a-history", - labels: map[string]string{"availabilityZone": az}, - wantErr: true, - }, - { - name: "cluster missing availabilityZone label", - obj: v1alpha1.History{ - Spec: v1alpha1.HistorySpec{ - AvailabilityZone: &az, - }, - }, - labels: map[string]string{}, - wantErr: true, - }, - { - name: "history missing availability zone (nil)", - obj: v1alpha1.History{ - Spec: v1alpha1.HistorySpec{}, - }, - labels: map[string]string{"availabilityZone": az}, - wantErr: true, - }, - { - name: "history missing availability zone (empty)", - obj: v1alpha1.History{ - Spec: v1alpha1.HistorySpec{ - AvailabilityZone: &empty, - }, - }, - labels: map[string]string{"availabilityZone": az}, - wantErr: true, - }, - { - name: "typed nil pointer doesn't panic", - obj: (*v1alpha1.History)(nil), - labels: map[string]string{"availabilityZone": az}, - wantErr: true, - }, - { - name: "nil object doesn't panic", - obj: nil, - labels: map[string]string{"availabilityZone": az}, - wantErr: true, - wantMatch: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - match, err := router.Match(tt.obj, tt.labels) - if tt.wantErr && err == nil { - t.Fatal("expected error, got nil") - } - if !tt.wantErr && err != nil { - t.Fatalf("unexpected error: %v", err) - } - if match != tt.wantMatch { - t.Errorf("expected match=%v, got %v", tt.wantMatch, match) - } - }) - } -} - func TestFlavorGroupCapacityResourceRouter_Match(t *testing.T) { router := FlavorGroupCapacityResourceRouter{} From 971821266c15cd77c9cee5ddfe3c281d5e5ccca2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:29:48 +0200 Subject: [PATCH 10/20] Renovate: Update External dependencies (#1377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Adoption](https://docs.renovatebot.com/merge-confidence/) | [Passing](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | Type | Update | |---|---|---|---|---|---|---|---| | [github.com/cobaltcore-dev/vm-operator](https://redirect.github.com/cobaltcore-dev/vm-operator) | `v0.2.8` → `v0.2.9` | ![age](https://developer.mend.io/api/mc/badges/age/go/github.com%2fcobaltcore-dev%2fvm-operator/v0.2.9?slim=true) | ![adoption](https://developer.mend.io/api/mc/badges/adoption/go/github.com%2fcobaltcore-dev%2fvm-operator/v0.2.9?slim=true) | ![passing](https://developer.mend.io/api/mc/badges/compatibility/go/github.com%2fcobaltcore-dev%2fvm-operator/v0.2.8/v0.2.9?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/go/github.com%2fcobaltcore-dev%2fvm-operator/v0.2.8/v0.2.9?slim=true) | require | patch | | [github.com/golangci/golangci-lint/v2](https://redirect.github.com/golangci/golangci-lint) | `v2.14.0` → `v2.15.0` | ![age](https://developer.mend.io/api/mc/badges/age/go/github.com%2fgolangci%2fgolangci-lint%2fv2/v2.15.0?slim=true) | ![adoption](https://developer.mend.io/api/mc/badges/adoption/go/github.com%2fgolangci%2fgolangci-lint%2fv2/v2.15.0?slim=true) | ![passing](https://developer.mend.io/api/mc/badges/compatibility/go/github.com%2fgolangci%2fgolangci-lint%2fv2/v2.14.0/v2.15.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/go/github.com%2fgolangci%2fgolangci-lint%2fv2/v2.14.0/v2.15.0?slim=true) | | minor | | [kube-prometheus-stack](https://redirect.github.com/prometheus-operator/kube-prometheus) ([source](https://redirect.github.com/prometheus-community/helm-charts)) | `92.9.0` → `92.10.0` | ![age](https://developer.mend.io/api/mc/badges/age/docker/ghcr.io%2fprometheus-community%2fcharts%2fkube-prometheus-stack/92.10.0?slim=true) | ![adoption](https://developer.mend.io/api/mc/badges/adoption/docker/ghcr.io%2fprometheus-community%2fcharts%2fkube-prometheus-stack/92.10.0?slim=true) | ![passing](https://developer.mend.io/api/mc/badges/compatibility/docker/ghcr.io%2fprometheus-community%2fcharts%2fkube-prometheus-stack/92.9.0/92.10.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/docker/ghcr.io%2fprometheus-community%2fcharts%2fkube-prometheus-stack/92.9.0/92.10.0?slim=true) | | minor | --- ### Release Notes
cobaltcore-dev/vm-operator (github.com/cobaltcore-dev/vm-operator) ### [`v0.2.9`](https://redirect.github.com/cobaltcore-dev/vm-operator/releases/tag/v0.2.9) [Compare Source](https://redirect.github.com/cobaltcore-dev/vm-operator/compare/v0.2.8...v0.2.9) **Full Changelog**:
golangci/golangci-lint (github.com/golangci/golangci-lint/v2) ### [`v2.15.0`](https://redirect.github.com/golangci/golangci-lint/blob/HEAD/CHANGELOG.md#v2150) [Compare Source](https://redirect.github.com/golangci/golangci-lint/compare/v2.14.0...v2.15.0) *Released on 2026-10-16* 1. Linters new features or changes - `errcheck`: from 1.20.0 to 1.30.0 - `exhaustive`: from 0.13.0 to 0.15.0 (⚠️ changes) - `gocognit`: from 1.2.1 to 1.3.0 - `golines`: from 0.15.0 to 0.16.0 - `govet-modernize`: from 0.50.0 to 0.52.0 - `loggercheck`: from 0.12.0 to 0.13.0 - `predeclared`: from 0.2.2 to 0.3.0 - `wrapcheck`: from 2.12.0 to 2.13.0 2. Linters bug fixes - `bidichk`: from 0.3.3 to 0.3.4 - `canonicalheader`: from [`a25c71c`](https://redirect.github.com/golangci/golangci-lint/commit/a25c71c521f6) to [`e18e5c5`](https://redirect.github.com/golangci/golangci-lint/commit/e18e5c5a078f) - `gofumpt`: improve deprecation logs - `gosec`: improve concurrency default value - `iface`: from 1.5.1 to 1.5.3 - `paralleltest`: from 1.0.15 to 1.0.16 - `revive`: from 1.17.0 to 1.17.1 - `revive`: the rules `context-keys-type`, `errorf`, `modifies-value-receiver`, `range-val-address`, `string-of-int`, `time-equal`, `time-naming`, `unexported-return`, `unhandled-error`, `var-declaration` are working (less randomly). - `rowserrcheck`: from [`c5f79b8`](https://redirect.github.com/golangci/golangci-lint/commit/c5f79b8a11ba) to [`e0740d5`](https://redirect.github.com/golangci/golangci-lint/commit/e0740d52dee8) - `staticcheck`: from 0.8.1 to 0.8.2 3. Bug fixes - fix: compute custom version hash with module versions - fix: env vars loading (related to `revive`) ⚠️ Changes of the `exhaustive` linter: - The replaced options still exists, and partially works, but they are deprecated. - The options `explicit-exhaustive-switch` and `explicit-exhaustive-map` have been replaced by `explicit-exhaustive`. - The `check` the value `map` has been replaced by `mapliteral`. - The option `ignore-enum-members` (string) has been replaced by `ignore-constants` (slice). - The option `ignore-enum-types` (string) has been replaced by `ignore-types` (slice). - New report messages. - The linter is a now a "slow" linter.
prometheus-community/helm-charts (kube-prometheus-stack) ### [`v92.10.0`](https://redirect.github.com/prometheus-community/helm-charts/releases/tag/kube-prometheus-stack-92.10.0) kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. #### What's Changed - \[kube-prometheus-stack] Update digest to [`f57ebcd`](https://redirect.github.com/prometheus-community/helm-charts/commit/f57ebcd) by [@​renovate](https://redirect.github.com/renovate)\[bot] in [#​7382](https://redirect.github.com/prometheus-community/helm-charts/pull/7382) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://redirect.github.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Makefile | 2 +- go.mod | 2 +- go.sum | 4 ++-- helm/dev/cortex-prometheus-operator/Chart.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 929e2b565..b6684d28a 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ GOLANGCI_LINT = $(LOCALBIN)/golangci-lint GOTESTSUM = $(LOCALBIN)/gotestsum CONTROLLER_TOOLS_VERSION ?= v0.23.0 -GOLANGCI_LINT_VERSION ?= v2.14.0 +GOLANGCI_LINT_VERSION ?= v2.15.0 GOTESTSUM_VERSION ?= v1.13.0 .PHONY: controller-gen diff --git a/go.mod b/go.mod index d94ba7649..6fe8d6601 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.27 require ( github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3 - github.com/cobaltcore-dev/vm-operator v0.2.8 + github.com/cobaltcore-dev/vm-operator v0.2.9 github.com/go-gorp/gorp v2.2.0+incompatible github.com/go-logr/logr v1.4.4 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index c44ebea7d..2d380f3ad 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3 h1:2KjrD+LzU8pgvm1HEIEw5qlRF1gcbFp7Q/t9z+AnY4g= github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3/go.mod h1:DyQ6MO1TsC4kzeG3l8tnIXQVeA/PITm22RyitJhDaKI= -github.com/cobaltcore-dev/vm-operator v0.2.8 h1:coH9Lptqxq6/M89wrfwUSJch30EYoZopKCrtasjUb58= -github.com/cobaltcore-dev/vm-operator v0.2.8/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4= +github.com/cobaltcore-dev/vm-operator v0.2.9 h1:1aSFkGY0VEyBkx+8xtY/JJQ9OHLY1CCIVYBbSVXhYPI= +github.com/cobaltcore-dev/vm-operator v0.2.9/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 2ce45c4f5..4137ffa19 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 92.9.0 + version: 92.10.0 From c8ddf4cb3ef428b706ea6ee7a6ff2453a0be6ee7 Mon Sep 17 00:00:00 2001 From: Marcel <156897072+mblos@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:47:47 +0200 Subject: [PATCH 11/20] feat(cr): prune aged committed-resource reservations (#1382) Reservations drift from their VM mapping and settle on invalid hosts over time. The syncer now prunes the oldest committed-resource Reservations past a max age (capped per run) before upserting, letting the reconciler rebuild fresh slots. Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com> --- helm/bundles/cortex-nova/values.yaml | 7 + .../reservations/commitments/syncer.go | 75 ++++++++- .../commitments/syncer_monitor.go | 11 ++ .../reservations/commitments/syncer_test.go | 153 +++++++++++++++++- 4 files changed, 244 insertions(+), 2 deletions(-) diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index b47acd8e6..c3fe2b356 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -350,6 +350,13 @@ cortex-scheduling-controllers: # Minimum age a CommittedResource CRD must reach before the syncer treats it as stale # and deletes it. Prevents timing races of recently-created CRDs not yet visible in the global fetch. staleCRDeletionGracePeriod: "10m" + # Age at which a committed-resource Reservation CRD is pruned so the reconciler re-creates a + # fresh slot. Reservations drift from their VM mapping over time and can end up on hosts whose + # labels no longer match. A zero value disables pruning. + staleReservationMaxAge: "168h" + # Maximum number of aged committed-resource Reservation CRDs pruned per syncer run, oldest first. + # A zero value disables pruning. + maxReservationDeletesPerRun: 20 cortex-knowledge-controllers: <<: *cortex diff --git a/internal/scheduling/reservations/commitments/syncer.go b/internal/scheduling/reservations/commitments/syncer.go index 6d95adc9d..1c0726289 100644 --- a/internal/scheduling/reservations/commitments/syncer.go +++ b/internal/scheduling/reservations/commitments/syncer.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "sort" "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -47,6 +48,14 @@ type SyncerConfig struct { // syncer deletes it as stale. Protects against race conditions where a CRD was just created // but the Limes response hasn't caught up yet. Defaults to 10 minutes if zero. StaleCRDeletionGracePeriod metav1.Duration `json:"staleCRDeletionGracePeriod,omitempty"` + // StaleReservationMaxAge is the age at which a committed-resource Reservation CRD is pruned so + // the reconciler re-creates a fresh slot. Reservations drift from their VM mapping over time + // and can end up on hosts whose labels no longer match; pruning the oldest resets them. + // A zero value disables age-based pruning. + StaleReservationMaxAge metav1.Duration `json:"staleReservationMaxAge,omitempty"` + // MaxReservationDeletesPerRun caps how many aged committed-resource Reservation CRDs the syncer + // prunes in a single run, oldest first. A zero value disables age-based pruning. + MaxReservationDeletesPerRun int `json:"maxReservationDeletesPerRun,omitempty"` // FlavorGroupResourceConfig maps flavor group names to resource configs; "*" acts as catch-all. // Not read from JSON — populated by the caller from the shared APIConfig. FlavorGroupResourceConfig map[string]FlavorGroupResourcesConfig @@ -277,6 +286,18 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { s.monitor.SetLimesCommitmentsActive(len(commitmentResult.states)) } + // Prune aged committed-resource Reservation CRDs before updating CommittedResources so the + // reconciler re-creates fresh slots. Reservations drift from their VM mapping over time and + // can end up on hosts whose labels no longer match. + agedReservationsPruned, err := s.pruneAgedReservations(ctx, logger) + if err != nil { + logger.Error(err, "failed to prune aged committed resource reservations") + if s.monitor != nil { + s.monitor.RecordError() + } + return err + } + // Upsert CommittedResource CRDs for each commitment var totalCreated, totalUpdated int for _, state := range commitmentResult.states { @@ -440,6 +461,9 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { if totalReservationDeleted > 0 { s.monitor.RecordOrphanReservationDeletes(totalReservationDeleted) } + if agedReservationsPruned > 0 { + s.monitor.RecordCRReservationPrunes(agedReservationsPruned) + } } if staleCRCount > 0 { @@ -455,10 +479,59 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { "staleCRs", staleCRCount, "staleDeleted", staleDeleted, "expiredCRsGCd", gcDeleted, - "orphanReservationsDeleted", totalReservationDeleted) + "orphanReservationsDeleted", totalReservationDeleted, + "agedReservationsPruned", agedReservationsPruned) return nil } +// pruneAgedReservations deletes committed-resource Reservation CRDs older than the configured +// max age, oldest first, up to the per-run cap. Reservations drift from their VM mapping over +// time and can settle on hosts whose labels no longer match; pruning them lets the reconciler +// re-create fresh slots. A zero max age or per-run cap disables pruning. Returns the number of +// Reservation CRDs deleted. +func (s *Syncer) pruneAgedReservations(ctx context.Context, logger logr.Logger) (int, error) { + maxAge := s.resourceConfig.StaleReservationMaxAge.Duration + maxDeletes := s.resourceConfig.MaxReservationDeletesPerRun + if maxAge <= 0 || maxDeletes <= 0 { + return 0, nil + } + + var reservations v1alpha1.ReservationList + if err := s.List(ctx, &reservations, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }); err != nil { + return 0, err + } + + // Collect reservations past the max age, then sort oldest-first so we prune the most-drifted + // slots first and stay within the per-run cap deterministically. + aged := make([]*v1alpha1.Reservation, 0, len(reservations.Items)) + for i := range reservations.Items { + res := &reservations.Items[i] + if time.Since(res.CreationTimestamp.Time) > maxAge { + aged = append(aged, res) + } + } + sort.Slice(aged, func(i, j int) bool { + return aged[i].CreationTimestamp.Before(&aged[j].CreationTimestamp) + }) + + deleted := 0 + for _, res := range aged { + if deleted >= maxDeletes { + break + } + if err := s.Delete(ctx, res); client.IgnoreNotFound(err) != nil { + logger.Error(err, "failed to prune aged committed resource reservation", "name", res.Name) + return deleted, err + } + logger.Info("pruned aged committed resource reservation", + "name", res.Name, "age", time.Since(res.CreationTimestamp.Time).Round(time.Second)) + deleted++ + } + return deleted, nil +} + func (s *Syncer) applyCommittedResourceSpec(cr *v1alpha1.CommittedResource, state *CommitmentState) { cr.Spec.CommitmentUUID = state.CommitmentUUID cr.Spec.SchedulingDomain = v1alpha1.SchedulingDomainNova diff --git a/internal/scheduling/reservations/commitments/syncer_monitor.go b/internal/scheduling/reservations/commitments/syncer_monitor.go index cd3862ce6..c206f8bee 100644 --- a/internal/scheduling/reservations/commitments/syncer_monitor.go +++ b/internal/scheduling/reservations/commitments/syncer_monitor.go @@ -28,6 +28,7 @@ type SyncerMonitor struct { crDeletes prometheus.Counter crStaleDeletes prometheus.Counter orphanReservationDeletes prometheus.Counter + crReservationPrunes prometheus.Counter } // NewSyncerMonitor creates a new monitor with Prometheus metrics. @@ -74,6 +75,10 @@ func NewSyncerMonitor() *SyncerMonitor { Name: "cortex_committed_resource_syncer_orphan_reservation_deletes_total", Help: "Total number of orphaned Reservation CRDs deleted by the syncer because their commitment is no longer active", }), + crReservationPrunes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "cortex_committed_resource_syncer_reservation_prunes_total", + Help: "Total number of aged committed-resource Reservation CRDs pruned by the syncer so the reconciler re-creates fresh slots", + }), } // Pre-initialize skip reason labels @@ -130,6 +135,10 @@ func (m *SyncerMonitor) RecordOrphanReservationDeletes(count int) { m.orphanReservationDeletes.Add(float64(count)) } +func (m *SyncerMonitor) RecordCRReservationPrunes(count int) { + m.crReservationPrunes.Add(float64(count)) +} + // Describe implements prometheus.Collector. func (m *SyncerMonitor) Describe(ch chan<- *prometheus.Desc) { m.syncErrors.Describe(ch) @@ -142,6 +151,7 @@ func (m *SyncerMonitor) Describe(ch chan<- *prometheus.Desc) { m.crDeletes.Describe(ch) m.crStaleDeletes.Describe(ch) m.orphanReservationDeletes.Describe(ch) + m.crReservationPrunes.Describe(ch) } // Collect implements prometheus.Collector. @@ -156,4 +166,5 @@ func (m *SyncerMonitor) Collect(ch chan<- prometheus.Metric) { m.crDeletes.Collect(ch) m.crStaleDeletes.Collect(ch) m.orphanReservationDeletes.Collect(ch) + m.crReservationPrunes.Collect(ch) } diff --git a/internal/scheduling/reservations/commitments/syncer_test.go b/internal/scheduling/reservations/commitments/syncer_test.go index 4ec12afe5..cb5f4fc5a 100644 --- a/internal/scheduling/reservations/commitments/syncer_test.go +++ b/internal/scheduling/reservations/commitments/syncer_test.go @@ -5,12 +5,14 @@ package commitments import ( "context" + "reflect" "sort" "testing" "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "k8s.io/apimachinery/pkg/api/resource" @@ -1179,7 +1181,8 @@ func TestSyncer_SyncReservations_OrphanReservationGC(t *testing.T) { orphanRes := &v1alpha1.Reservation{ ObjectMeta: metav1.ObjectMeta{ - Name: "orphan-reservation", + Name: "orphan-reservation", + CreationTimestamp: metav1.NewTime(time.Now()), Labels: map[string]string{ v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, }, @@ -1257,3 +1260,151 @@ func TestSyncer_SyncReservations_OrphanReservationGC(t *testing.T) { }) } } + +func TestSyncer_pruneAgedReservations(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add scheme: %v", err) + } + + // makeReservation builds a committed-resource Reservation with the given name and age. + makeReservation := func(name string, age time.Duration) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + CreationTimestamp: metav1.NewTime(time.Now().Add(-age)), + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource, + }, + }, + Spec: v1alpha1.ReservationSpec{ + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + CommitmentUUID: name, + }, + }, + } + } + + const day = 24 * time.Hour + + tests := []struct { + name string + maxAge time.Duration + maxDeletes int + reservations []*v1alpha1.Reservation + wantDeleted int + wantRemain []string + }{ + { + name: "all young: nothing deleted", + maxAge: 7 * day, + maxDeletes: 20, + reservations: []*v1alpha1.Reservation{ + makeReservation("res-a", 1*day), + makeReservation("res-b", 2*day), + }, + wantDeleted: 0, + wantRemain: []string{"res-a", "res-b"}, + }, + { + name: "zero max age: pruning disabled", + maxAge: 0, + maxDeletes: 20, + reservations: []*v1alpha1.Reservation{ + makeReservation("res-old", 10*day), + makeReservation("res-oldest", 30*day), + }, + wantDeleted: 0, + wantRemain: []string{"res-old", "res-oldest"}, + }, + { + name: "zero per-run cap: pruning disabled", + maxAge: 7 * day, + maxDeletes: 0, + reservations: []*v1alpha1.Reservation{ + makeReservation("res-old", 10*day), + makeReservation("res-oldest", 30*day), + }, + wantDeleted: 0, + wantRemain: []string{"res-old", "res-oldest"}, + }, + { + name: "oldest-first deletion past threshold", + maxAge: 7 * day, + maxDeletes: 20, + reservations: []*v1alpha1.Reservation{ + makeReservation("res-young", 1*day), + makeReservation("res-old", 10*day), + makeReservation("res-oldest", 30*day), + }, + wantDeleted: 2, + wantRemain: []string{"res-young"}, + }, + { + name: "max-count cap respected, oldest deleted first", + maxAge: 7 * day, + maxDeletes: 1, + reservations: []*v1alpha1.Reservation{ + makeReservation("res-old", 10*day), + makeReservation("res-oldest", 30*day), + }, + wantDeleted: 1, + wantRemain: []string{"res-old"}, // oldest (res-oldest) pruned first + }, + { + name: "max-age threshold respected", + maxAge: 7 * day, + maxDeletes: 20, + reservations: []*v1alpha1.Reservation{ + makeReservation("res-6d", 6*day), + makeReservation("res-8d", 8*day), + }, + wantDeleted: 1, + wantRemain: []string{"res-6d"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + objs := make([]client.Object, 0, len(tc.reservations)) + for _, r := range tc.reservations { + objs = append(objs, r) + } + k8sClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objs...). + Build() + + syncer := &Syncer{ + Client: k8sClient, + resourceConfig: SyncerConfig{ + StaleReservationMaxAge: metav1.Duration{Duration: tc.maxAge}, + MaxReservationDeletesPerRun: tc.maxDeletes, + }, + } + + deleted, err := syncer.pruneAgedReservations(context.Background(), logr.Discard()) + if err != nil { + t.Fatalf("pruneAgedReservations() error = %v", err) + } + if deleted != tc.wantDeleted { + t.Errorf("deleted = %d, want %d", deleted, tc.wantDeleted) + } + + var remaining v1alpha1.ReservationList + if err := k8sClient.List(context.Background(), &remaining); err != nil { + t.Fatalf("Failed to list reservations: %v", err) + } + gotNames := make([]string, 0, len(remaining.Items)) + for i := range remaining.Items { + gotNames = append(gotNames, remaining.Items[i].Name) + } + sort.Strings(gotNames) + want := append([]string(nil), tc.wantRemain...) + sort.Strings(want) + if !reflect.DeepEqual(gotNames, want) { + t.Errorf("remaining = %v, want %v", gotNames, want) + } + }) + } +} From debd8750f1d7d6e7874e9437cc23609d379481d3 Mon Sep 17 00:00:00 2001 From: "cortex-ai-agents[bot]" <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:10:47 +0200 Subject: [PATCH 12/20] =?UTF-8?q?docs:=20correct=20Decision=20CRD=20sectio?= =?UTF-8?q?n=20=E2=80=94=20Decision=20is=20transient,=20not=20persisted=20?= =?UTF-8?q?(#1381)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - **Corrects the "Inspecting decisions and descheduling" section** in `docs/02-external-scheduler-api/02-nova-smart-scheduling.md` to reflect that the `Decision` CRD is a transient in-memory object, not persisted to etcd (as documented in `api/v1alpha1/decision_types.go` lines 126-132) - **Replaces the misleading `kubectl get decisions` example** (which would return no results) with a `kubectl get virtualmachines` command showing the `spec.cortex.explanation`, `spec.cortex.intent`, and `spec.cortex.hostCandidates` fields — the actual persisted scheduling results - **Keeps the `kubectl get deschedulings` example** unchanged, since `Descheduling` resources are persisted and the example is accurate ## Test plan - [ ] Verify the renamed heading anchor (`#inspecting-scheduling-results`) does not break any cross-links in the docs (confirmed: no files linked to the old anchor) - [ ] Confirm the `spec.cortex.explanation`, `spec.cortex.hostCandidates`, and `spec.cortex.intent` field paths match the code in `internal/scheduling/nova/vm_handler.go` - [ ] Confirm the `Descheduling` print columns in the kept example still match `api/v1alpha1/descheduling_types.go` - [ ] Read the section in context to verify it flows naturally between "Closing the loop: descheduling" and the closing paragraph / `## Next` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 --- .../02-nova-smart-scheduling.md | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md index 341f2b107..b1540706f 100644 --- a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md +++ b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md @@ -137,24 +137,27 @@ knowledge feature) exceeds a threshold over an observed window. The flow: issue real live-migrations via the Nova client). 4. A **cleanup** controller TTL-expires old `Descheduling` resources (and runs once on startup). -## Inspecting decisions and descheduling +## Inspecting scheduling results -Two cluster-scoped CRDs record what the pipelines did — inspect them with `kubectl` (no namespace, -Cortex kinds are cluster-scoped). Each request that hits the external scheduler produces a **`Decision`**: +Each scheduling request produces a **`Decision`** internally, but the Decision is a transient +in-memory object — it is **not** persisted to etcd. (The `Decision` CRD exists in +`api/v1alpha1/`; its long-term shape is under discussion, but currently `kubectl get decisions` +returns no results.) Instead, the scheduling outcome is written onto the **`VirtualMachine`** +resource (the vm-operator CRD, cluster-scoped) so operators can inspect it directly: ```bash -kubectl get decisions +kubectl get virtualmachines -o custom-columns=\ +NAME:.metadata.name,\ +EXPLANATION:.spec.cortex.explanation,\ +INTENT:.spec.cortex.intent,\ +HOST:.spec.cortex.hostCandidates[0].name ``` -``` -DOMAIN RESOURCE ID # CREATED PIPELINE TARGETHOST READY -nova a1b2c3… 1 2m kvm-general-purpose-load-balancing node-042 True -``` - -`TargetHost` (`.status.result.targetHost`) is what Cortex recommended and `Pipeline` names the pipeline -that ran; `-o yaml` shows the full filter/weigher trace, and `.status.explanation` carries a -human-readable summary of why that host won. The detector pipeline emits **`Descheduling`** -resources for VMs it wants to move: +`.spec.cortex.explanation` carries a human-readable summary of why the chosen host won, +`.spec.cortex.intent` records the scheduling intent (e.g. initial placement or evacuation), and +`.spec.cortex.hostCandidates` lists the candidate hosts in ranked order; `-o yaml` on a single +VM shows the full detail. The detector pipeline emits **`Descheduling`** resources for VMs it +wants to move: ```bash kubectl get deschedulings @@ -165,14 +168,12 @@ PREVIOUS HOST NEW HOST CREATED REASON(S) READY node-042 node-017 30s avoid_high_steal_pct True ``` -The columns come from each resource's status; the field-by-field definitions live on the `Decision` -and `Descheduling` Go types in `api/v1alpha1/`. For VM placements the same human-readable explanation -is also written onto the scheduled VM's `spec.cortex.explanation` (the vm-operator `VirtualMachine`), -so operators can trace a placement from the VM itself. +The columns come from the resource's status; the field-by-field definitions live on the +`Descheduling` Go type in `api/v1alpha1/`. Nova exercises the full arc: an HTTP request drives a filter-weigher pipeline whose weighers read -knowledge features and respect reservations, the result is a `Decision` (with its explanation recorded -on the scheduled VM), committed slots are recorded, and a scheduled detector emits `Descheduling` +knowledge features and respect reservations, the result is recorded on the scheduled VM's +`spec.cortex` fields, committed slots are recorded, and a scheduled detector emits `Descheduling` recommendations. Every other domain in this chapter is a smaller version of this same shape — starting with the storage domains, which are HTTP-driven like Nova but ship far fewer steps. From bdcadcc902d673b939399f65179a76a5fbca7932 Mon Sep 17 00:00:00 2001 From: "cortex-ai-agents[bot]" <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:49:35 +0200 Subject: [PATCH 13/20] fix: use decision intent in VM handler instead of re-deriving from request (#1379) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Fixes a bug where `VirtualMachineHandler.Upsert()` would fail on every initial VM create because it re-called `request.GetIntent()` which returns an error when Nova doesn't set the `_nova_check_type` scheduler hint (the case for initial placements) - The API layer already handles this gracefully by defaulting to `CreateIntent` and storing it on `decision.Spec.Intent` — the VM handler now reads from that field instead of re-deriving - Updates the test helper `vmTestDecision()` to include `Spec.Intent`, matching the real data flow ## Context This was introduced in #1362 (Create VM resources as part of nova pipeline) and partially addressed in #1374 and #1376 (intent fixes), but the VM handler still had its own `GetIntent()` call that didn't benefit from the API's fallback logic. ## Test plan - [ ] Existing `vm_handler_test.go` tests pass with the updated decision helper - [ ] Verify that initial VM creates no longer fail when `VirtualMachineCreation` feature gate is enabled 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 --- internal/scheduling/nova/vm_handler.go | 10 ++++------ internal/scheduling/nova/vm_handler_test.go | 11 ++++++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/internal/scheduling/nova/vm_handler.go b/internal/scheduling/nova/vm_handler.go index 957b50503..3548122f3 100644 --- a/internal/scheduling/nova/vm_handler.go +++ b/internal/scheduling/nova/vm_handler.go @@ -136,11 +136,9 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.D } } - // Needed to populate the Cortex intent field and the status condition. - intent, err := request.GetIntent() - if err != nil { - return fmt.Errorf("failed to get intent: %w", err) - } + // Read intent from the decision, which was already resolved by the API layer + // (including a graceful fallback to CreateIntent for initial placements). + intent := decision.Spec.Intent // Write the cortex spec (our scheduling result). vm.Spec.Cortex = &vmv1alpha1.CortexSpec{} @@ -181,7 +179,7 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, decision *v1alpha1.D // If the vm resource doesn't exist yet, create it. vmInCluster := &vmv1alpha1.VirtualMachine{} - err = h.Get(ctx, types.NamespacedName{ + err := h.Get(ctx, types.NamespacedName{ Namespace: vm.Namespace, Name: vm.Name, }, vmInCluster) diff --git a/internal/scheduling/nova/vm_handler_test.go b/internal/scheduling/nova/vm_handler_test.go index e4d5b543e..8c1499c07 100644 --- a/internal/scheduling/nova/vm_handler_test.go +++ b/internal/scheduling/nova/vm_handler_test.go @@ -57,9 +57,14 @@ func vmTestRequest() api.ExternalSchedulerRequest { } func vmTestDecision(targetHost *string) *v1alpha1.Decision { - return &v1alpha1.Decision{Status: v1alpha1.DecisionStatus{ - Result: &v1alpha1.DecisionResult{TargetHost: targetHost}, - }} + return &v1alpha1.Decision{ + Spec: v1alpha1.DecisionSpec{ + Intent: api.CreateIntent, + }, + Status: v1alpha1.DecisionStatus{ + Result: &v1alpha1.DecisionResult{TargetHost: targetHost}, + }, + } } func vmTestHypervisor(region, az string) *hv1.Hypervisor { From e168eb5a291e2b01d4b9e6567acad93ea2c5a90a Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 19 Oct 2026 10:51:27 +0200 Subject: [PATCH 14/20] chore: Add edge cases to explanation (#1383) Signed-off-by: Markus Wieland --- internal/scheduling/lib/explanation.go | 11 +++++++---- internal/scheduling/lib/explanation_test.go | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/internal/scheduling/lib/explanation.go b/internal/scheduling/lib/explanation.go index 3506f8a54..c1deeff0f 100644 --- a/internal/scheduling/lib/explanation.go +++ b/internal/scheduling/lib/explanation.go @@ -35,11 +35,14 @@ func GenerateExplanation(result *v1alpha1.DecisionResult, pipelineErr error) str return fmt.Sprintf("Pipeline run failed: %s.", pipelineErr.Error()) } - if result == nil || len(result.StepResults) == 0 { - if result != nil && result.TargetHost != nil { + if result == nil { + return "No decision result recorded." + } + if len(result.StepResults) == 0 { + if result.TargetHost != nil { return fmt.Sprintf("Selected host: %s.", *result.TargetHost) } - return "" + return "No pipeline steps recorded." } // Get all initial hosts from input weights. @@ -53,7 +56,7 @@ func GenerateExplanation(result *v1alpha1.DecisionResult, pipelineErr error) str if result.TargetHost != nil { return fmt.Sprintf("Selected host: %s.", *result.TargetHost) } - return "" + return "No host weight data recorded." } var sb strings.Builder diff --git a/internal/scheduling/lib/explanation_test.go b/internal/scheduling/lib/explanation_test.go index 25fcf075f..2c020f396 100644 --- a/internal/scheduling/lib/explanation_test.go +++ b/internal/scheduling/lib/explanation_test.go @@ -24,7 +24,7 @@ func TestGenerateExplanation(t *testing.T) { name: "nil result no error", result: nil, err: nil, - expected: "", + expected: "No decision result recorded.", }, { name: "nil result with error", @@ -97,7 +97,7 @@ func TestGenerateExplanation(t *testing.T) { {StepName: "some-step", Activations: map[string]float64{}}, }, }, - expected: "", + expected: "No host weight data recorded.", }, { name: "no weights with target", From c6824705961f15a61dbdac08e9cb9d47891e63fc Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 08:53:00 +0000 Subject: [PATCH 15/20] Renovate: Update kube-prometheus-stack Docker tag to v92.10.1 (#1378) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | [kube-prometheus-stack](https://redirect.github.com/prometheus-operator/kube-prometheus) ([source](https://redirect.github.com/prometheus-community/helm-charts)) | patch | `92.10.0` → `92.10.1` | --- ### Release Notes
prometheus-community/helm-charts (kube-prometheus-stack) ### [`v92.10.1`](https://redirect.github.com/prometheus-community/helm-charts/releases/tag/kube-prometheus-stack-92.10.1) [Compare Source](https://redirect.github.com/prometheus-community/helm-charts/compare/kube-prometheus-stack-92.10.0...kube-prometheus-stack-92.10.1) kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. #### What's Changed - \[kube-prometheus-stack] Update grafana Docker tag to v14.0.1 by [@​renovate](https://redirect.github.com/renovate)\[bot] in [#​7383](https://redirect.github.com/prometheus-community/helm-charts/pull/7383) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- helm/dev/cortex-prometheus-operator/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 4137ffa19..8805a7699 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 92.10.0 + version: 92.10.1 From 335e6a5b0ea4942b918876bee188b2bdf282abdb Mon Sep 17 00:00:00 2001 From: Marcel <156897072+mblos@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:53:46 +0200 Subject: [PATCH 16/20] fix: change execution order (#1384) Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com> --- .../reservations/commitments/syncer.go | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/scheduling/reservations/commitments/syncer.go b/internal/scheduling/reservations/commitments/syncer.go index 1c0726289..163032ee1 100644 --- a/internal/scheduling/reservations/commitments/syncer.go +++ b/internal/scheduling/reservations/commitments/syncer.go @@ -272,32 +272,32 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { return err } - // Get all commitments as states - commitmentResult, err := s.getCommitmentStates(ctx, logger, flavorGroups) + // Prune aged committed-resource Reservation CRDs before updating CommittedResources so the + // reconciler re-creates fresh slots. Reservations drift from their VM mapping over time and + // can end up on hosts whose labels no longer match. + agedReservationsPruned, err := s.pruneAgedReservations(ctx, logger) if err != nil { - logger.Error(err, "failed to get compute commitments") + logger.Error(err, "failed to prune aged committed resource reservations") if s.monitor != nil { s.monitor.RecordError() } return err } - if s.monitor != nil { - s.monitor.SetLimesCommitmentsActive(len(commitmentResult.states)) - } - - // Prune aged committed-resource Reservation CRDs before updating CommittedResources so the - // reconciler re-creates fresh slots. Reservations drift from their VM mapping over time and - // can end up on hosts whose labels no longer match. - agedReservationsPruned, err := s.pruneAgedReservations(ctx, logger) + // Get all commitments as states + commitmentResult, err := s.getCommitmentStates(ctx, logger, flavorGroups) if err != nil { - logger.Error(err, "failed to prune aged committed resource reservations") + logger.Error(err, "failed to get compute commitments") if s.monitor != nil { s.monitor.RecordError() } return err } + if s.monitor != nil { + s.monitor.SetLimesCommitmentsActive(len(commitmentResult.states)) + } + // Upsert CommittedResource CRDs for each commitment var totalCreated, totalUpdated int for _, state := range commitmentResult.states { From 4eb4491e373d4521b1a1bf03ba82e982eef19536 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:27:33 +0200 Subject: [PATCH 17/20] bump app version [skip ci] (#1353) bump app version [skip ci] ``` bumped cortex: sha-01f5a42b -> sha-c6824705 bumped cortex-shim: sha-f73e47fa -> sha-97182126 noop cortex-postgres (sha-dbe244dd) ``` Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- helm/library/cortex-shim/Chart.yaml | 2 +- helm/library/cortex/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index 971de5e6b..8ae6fd5cb 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -3,6 +3,6 @@ name: cortex-shim description: A Helm chart to distribute cortex shims. type: application version: 0.1.32 -appVersion: "sha-f73e47fa" +appVersion: "sha-97182126" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index 31edd9ffd..bb49b2b0b 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -3,6 +3,6 @@ name: cortex description: A Helm chart to distribute cortex. type: application version: 0.6.8 -appVersion: "sha-01f5a42b" +appVersion: "sha-c6824705" icon: "https://example.com/icon.png" dependencies: [] From b4cf3be3a5cfe82c0fb541cb2c885cfabb2ec3fa Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Mon, 19 Oct 2026 11:40:11 +0200 Subject: [PATCH 18/20] fix: update Perses dashboard annotation to label in alerts (#1370) - Changed the documentation in values.yaml for cortex-nova and cortex-placement-shim to reflect that the Perses dashboard link is emitted as a label instead of an annotation. - Updated the alerts.yaml files in cortex-placement-shim to use the new `persesDashboard` label format for alert definitions, ensuring consistency with the updated values.yaml documentation. Signed-off-by: Markus Wieland --- .../cortex-cinder/templates/alerts.yaml | 90 +++--- helm/bundles/cortex-cinder/values.yaml | 4 +- .../cortex-manila/templates/alerts.yaml | 78 ++--- helm/bundles/cortex-manila/values.yaml | 4 +- .../bundles/cortex-nova/templates/alerts.yaml | 270 +++++++++--------- helm/bundles/cortex-nova/values.yaml | 4 +- .../templates/alerts.yaml | 66 ++--- .../bundles/cortex-placement-shim/values.yaml | 4 +- 8 files changed, 260 insertions(+), 260 deletions(-) diff --git a/helm/bundles/cortex-cinder/templates/alerts.yaml b/helm/bundles/cortex-cinder/templates/alerts.yaml index ecd7bd6bd..afea6c8e9 100644 --- a/helm/bundles/cortex-cinder/templates/alerts.yaml +++ b/helm/bundles/cortex-cinder/templates/alerts.yaml @@ -23,15 +23,15 @@ spec: absent(up{pod=~"cortex-cinder-scheduling-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: liveness service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/down annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Cortex Scheduling for Cinder is down" description: > The Cortex scheduling service is down. Scheduling requests from Cinder will @@ -44,15 +44,15 @@ spec: absent(up{pod=~"cortex-cinder-knowledge-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: liveness service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/down annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Cortex Knowledge for Cinder is down" description: > The Cortex Knowledge service is down. This is no immediate problem, @@ -63,14 +63,14 @@ spec: expr: rate(cortex_scheduler_api_request_duration_seconds_count{service="cortex-cinder-metrics", status=~"4.+"}[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: api service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Cinder Scheduler HTTP request 400 errors too high" description: > Cinder Scheduler is responding to placement requests with HTTP 4xx @@ -82,14 +82,14 @@ spec: expr: rate(cortex_scheduler_api_request_duration_seconds_count{service="cortex-cinder-metrics", status=~"5.+" }[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: api service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Cinder Scheduler HTTP request 500 errors too high" description: > Cinder Scheduler is responding to placement requests with HTTP 5xx errors. @@ -113,15 +113,15 @@ spec: ({{ .Values.alerts.thresholds.highMemoryFraction }} * kube_pod_container_resource_limits{resource="memory", container="manager", pod=~"cortex-cinder-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: memory service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/deployment annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` uses too much memory" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is using more than {{ mulf .Values.alerts.thresholds.highMemoryFraction 100 | int }}% of its configured @@ -141,15 +141,15 @@ spec: ({{ .Values.alerts.thresholds.highCPUFraction }} * kube_pod_container_resource_limits{resource="cpu", container="manager", pod=~"cortex-cinder-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: cpu service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/deployment annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` uses too much CPU" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is using more than {{ mulf .Values.alerts.thresholds.highCPUFraction 100 | int }}% of its configured @@ -161,14 +161,14 @@ spec: expr: rate(cortex_db_connection_attempts_total{service="cortex-cinder-metrics"}[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: db service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` is trying to connect to the database too often" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is trying to connect to the database too often. This may happen @@ -178,14 +178,14 @@ spec: expr: cortex_sync_request_processed_total{service="cortex-cinder-metrics"} - cortex_sync_request_duration_seconds_count{service="cortex-cinder-metrics"} > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: syncstatus service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` Sync not successful" description: > `{{ "{{" }} $labels.component {{ "}}" }}` experienced an issue syncing data from the datasource `{{ "{{" }} $labels.datasource {{ "}}" }}`. This may @@ -199,14 +199,14 @@ spec: expr: cortex_sync_objects{service="cortex-cinder-metrics"} == 0 and (cortex_sync_objects{service="cortex-cinder-metrics"} offset 1h > 0) for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: syncobjects service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` is not syncing any new data from `{{ "{{" }} $labels.datasource {{ "}}" }}`" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is not syncing any objects from the datasource `{{ "{{" }} $labels.datasource {{ "}}" }}`. This may happen @@ -220,14 +220,14 @@ spec: expr: cortex_datasource_state{domain="cinder",state!="ready"} != 0 for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: datasources service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Datasource `{{ "{{" }} $labels.datasource {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the datasource @@ -238,14 +238,14 @@ spec: expr: cortex_knowledge_state{domain="cinder",state!="ready"} != 0 for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: knowledge service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Knowledge `{{ "{{" }} $labels.knowledge {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the knowledge @@ -256,14 +256,14 @@ spec: expr: cortex_decision_state{domain="cinder",state="error"} > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: decisions service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Some decisions are in error state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" description: > The cortex scheduling pipeline generated decisions that are in error state. @@ -275,14 +275,14 @@ spec: expr: cortex_decision_state{domain="cinder",state="waiting"} > 10 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: decisions service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Too many decisions are in waiting state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" description: > The cortex scheduling pipeline has a high number of decisions for which @@ -297,14 +297,14 @@ spec: cortex_kpi_state{domain="cinder",state!="ready"} != 0 for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: kpis service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "KPI `{{ "{{" }} $labels.kpi {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the KPI @@ -315,14 +315,14 @@ spec: expr: cortex_pipeline_state{domain="cinder",state!="ready"} != 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: pipelines service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Pipeline `{{ "{{" }} $labels.pipeline {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the pipeline diff --git a/helm/bundles/cortex-cinder/values.yaml b/helm/bundles/cortex-cinder/values.yaml index a4873c3ef..b11cbfc38 100644 --- a/helm/bundles/cortex-cinder/values.yaml +++ b/helm/bundles/cortex-cinder/values.yaml @@ -18,11 +18,11 @@ alerts: enabled: true prometheus: openstack # Base URL (no trailing slash) for Perses dashboard links emitted as the - # `persesDashboard` annotation on each alert. The per-alert dashboard slug is + # `persesDashboard` label on each alert. The per-alert dashboard slug is # appended to it. The value is templateable with Prometheus label expressions, # e.g. a region-specific host: # https://perses.{{ $labels.region }}.example.com/projects/observability/dashboards - # Left empty by default so no persesDashboard annotation is emitted. + # Left empty by default so no persesDashboard label is emitted. persesDashboardBaseURL: "" thresholds: # Fraction of each pod's configured container memory limit above which diff --git a/helm/bundles/cortex-manila/templates/alerts.yaml b/helm/bundles/cortex-manila/templates/alerts.yaml index 234dc0a5c..f78829870 100644 --- a/helm/bundles/cortex-manila/templates/alerts.yaml +++ b/helm/bundles/cortex-manila/templates/alerts.yaml @@ -23,15 +23,15 @@ spec: absent(up{pod=~"cortex-manila-scheduling-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: liveness service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/down annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Cortex Scheduling for Manila is down" description: > The Cortex scheduling service is down. Scheduling requests from Manila will @@ -44,15 +44,15 @@ spec: absent(up{pod=~"cortex-manila-knowledge-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: liveness service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/down annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Cortex Knowledge for Manila is down" description: > The Cortex Knowledge service is down. This is no immediate problem, @@ -63,15 +63,15 @@ spec: expr: rate(cortex_scheduler_api_request_duration_seconds_count{service="cortex-manila-metrics", status=~"4.+"}[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/apierrors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Manila Scheduler HTTP request 400 errors too high" description: > Manila Scheduler is responding to placement requests with HTTP 4xx @@ -83,15 +83,15 @@ spec: expr: rate(cortex_scheduler_api_request_duration_seconds_count{service="cortex-manila-metrics", status=~"5.+" }[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/apierrors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Manila Scheduler HTTP request 500 errors too high" description: > Manila Scheduler is responding to placement requests with HTTP 5xx errors. @@ -115,15 +115,15 @@ spec: ({{ .Values.alerts.thresholds.highMemoryFraction }} * kube_pod_container_resource_limits{resource="memory", container="manager", pod=~"cortex-manila-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: memory service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/deployment annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` uses too much memory" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is using more than {{ mulf .Values.alerts.thresholds.highMemoryFraction 100 | int }}% of its configured @@ -143,15 +143,15 @@ spec: ({{ .Values.alerts.thresholds.highCPUFraction }} * kube_pod_container_resource_limits{resource="cpu", container="manager", pod=~"cortex-manila-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: cpu service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/deployment annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` uses too much CPU" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is using more than {{ mulf .Values.alerts.thresholds.highCPUFraction 100 | int }}% of its configured @@ -163,15 +163,15 @@ spec: expr: rate(cortex_db_connection_attempts_total{service="cortex-manila-metrics"}[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: db service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/database annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` is trying to connect to the database too often" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is trying to connect to the database too often. This may happen @@ -181,15 +181,15 @@ spec: expr: cortex_sync_request_processed_total{service="cortex-manila-metrics"} - cortex_sync_request_duration_seconds_count{service="cortex-manila-metrics"} > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: syncstatus service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/datasources annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` Sync not successful" description: > `{{ "{{" }} $labels.component {{ "}}" }}` experienced an issue syncing data from the datasource `{{ "{{" }} $labels.datasource {{ "}}" }}`. This may @@ -203,15 +203,15 @@ spec: expr: cortex_sync_objects{service="cortex-manila-metrics"} == 0 and (cortex_sync_objects{service="cortex-manila-metrics"} offset 1h > 0) for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: syncobjects service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/datasources annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` is not syncing any new data from `{{ "{{" }} $labels.datasource {{ "}}" }}`" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is not syncing any objects from the datasource `{{ "{{" }} $labels.datasource {{ "}}" }}`. This may happen @@ -225,15 +225,15 @@ spec: expr: cortex_datasource_state{domain="manila",state!="ready"} != 0 for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: datasources service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/unready annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Datasource `{{ "{{" }} $labels.datasource {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the datasource @@ -244,15 +244,15 @@ spec: expr: cortex_knowledge_state{domain="manila",state!="ready"} != 0 for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: knowledge service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/unready annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Knowledge `{{ "{{" }} $labels.knowledge {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the knowledge @@ -264,15 +264,15 @@ spec: cortex_kpi_state{domain="manila",state!="ready"} != 0 for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: kpis service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/unready annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "KPI `{{ "{{" }} $labels.kpi {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the KPI @@ -283,15 +283,15 @@ spec: expr: cortex_pipeline_state{domain="manila",state!="ready"} != 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: kpis service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/unready annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Pipeline `{{ "{{" }} $labels.pipeline {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the pipeline diff --git a/helm/bundles/cortex-manila/values.yaml b/helm/bundles/cortex-manila/values.yaml index 34a8c9702..5d48da8f8 100644 --- a/helm/bundles/cortex-manila/values.yaml +++ b/helm/bundles/cortex-manila/values.yaml @@ -18,11 +18,11 @@ alerts: enabled: true prometheus: openstack # Base URL (no trailing slash) for Perses dashboard links emitted as the - # `persesDashboard` annotation on each alert. The per-alert dashboard slug is + # `persesDashboard` label on each alert. The per-alert dashboard slug is # appended to it. The value is templateable with Prometheus label expressions, # e.g. a region-specific host: # https://perses.{{ $labels.region }}.example.com/projects/observability/dashboards - # Left empty by default so no persesDashboard annotation is emitted. + # Left empty by default so no persesDashboard label is emitted. persesDashboardBaseURL: "" thresholds: # Fraction of each pod's configured container memory limit above which diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index 994bd8b92..9db87a9cc 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -23,15 +23,15 @@ spec: absent(up{pod=~"cortex-nova-scheduling-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: liveness service: cortex severity: {{ if and .Values.kvm.enabled .Values.kvm.criticalAlerts }}critical{{ else }}warning{{ end }} support_group: workload-management playbook: docs/support/playbook/cortex/alerts/down annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Cortex Scheduling for Nova is down" description: > The Cortex scheduling service is down. Scheduling requests from Nova will @@ -45,15 +45,15 @@ spec: absent(up{pod=~"cortex-nova-knowledge-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: liveness service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/down annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Cortex Knowledge for Nova is down" description: > The Cortex Knowledge service is down. This is no immediate problem, @@ -64,14 +64,14 @@ spec: expr: delta(cortex_detector_pipeline_run_duration_seconds_count{component="nova-scheduling", error="true"}[2m]) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: descheduler service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Descheduler pipeline is erroring." description: > The Cortex descheduler pipeline is encountering errors during its execution. @@ -82,15 +82,15 @@ spec: expr: rate(cortex_scheduler_api_request_duration_seconds_count{service="cortex-nova-metrics", status=~"4.+"}[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Nova Scheduler HTTP request 400 errors too high" description: > Nova Scheduler is responding to placement requests with HTTP 4xx @@ -102,15 +102,15 @@ spec: expr: rate(cortex_scheduler_api_request_duration_seconds_count{service="cortex-nova-metrics", status=~"5.+" }[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Nova Scheduler HTTP request 500 errors too high" description: > Nova Scheduler is responding to placement requests with HTTP 5xx errors. @@ -134,15 +134,15 @@ spec: ({{ .Values.alerts.thresholds.highMemoryFraction }} * kube_pod_container_resource_limits{resource="memory", container="manager", pod=~"cortex-nova-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: memory service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/deployment annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` uses too much memory" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is using more than {{ mulf .Values.alerts.thresholds.highMemoryFraction 100 | int }}% of its configured @@ -162,15 +162,15 @@ spec: ({{ .Values.alerts.thresholds.highCPUFraction }} * kube_pod_container_resource_limits{resource="cpu", container="manager", pod=~"cortex-nova-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: cpu service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/deployment annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` uses too much CPU" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is using more than {{ mulf .Values.alerts.thresholds.highCPUFraction 100 | int }}% of its configured @@ -182,15 +182,15 @@ spec: expr: rate(cortex_db_connection_attempts_total{service="cortex-nova-metrics"}[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: db service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/database annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` is trying to connect to the database too often" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is trying to connect to the database too often. This may happen @@ -200,15 +200,15 @@ spec: expr: cortex_sync_request_processed_total{service="cortex-nova-metrics"} - cortex_sync_request_duration_seconds_count{service="cortex-nova-metrics"} > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: syncstatus service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/datasources annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` Sync not successful" description: > `{{ "{{" }} $labels.component {{ "}}" }}` experienced an issue syncing data from the datasource `{{ "{{" }} $labels.datasource {{ "}}" }}`. This may @@ -222,15 +222,15 @@ spec: expr: cortex_sync_objects{service="cortex-nova-metrics", datasource!="openstack_migrations"} == 0 and (cortex_sync_objects{service="cortex-nova-metrics", datasource!="openstack_migrations"} offset 1h > 0) for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: syncobjects service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/datasources annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "`{{ "{{" }} $labels.component {{ "}}" }}` is not syncing any new data from `{{ "{{" }} $labels.datasource {{ "}}" }}`" description: > `{{ "{{" }} $labels.component {{ "}}" }}` is not syncing any objects from the datasource `{{ "{{" }} $labels.datasource {{ "}}" }}`. This may happen @@ -244,15 +244,15 @@ spec: expr: cortex_datasource_state{domain="nova",state!="ready"} != 0 for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: datasources service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/unready annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Datasource `{{ "{{" }} $labels.datasource {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the datasource @@ -263,15 +263,15 @@ spec: expr: cortex_knowledge_state{domain="nova",state!="ready"} != 0 for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: knowledge service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/unready annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Knowledge `{{ "{{" }} $labels.knowledge {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the knowledge @@ -282,14 +282,14 @@ spec: expr: cortex_decision_state{domain="nova",state="error"} > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: decisions service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Some decisions are in error state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" description: > The cortex scheduling pipeline generated decisions that are in error state. @@ -301,14 +301,14 @@ spec: expr: cortex_decision_state{domain="nova",state="waiting"} > 10 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: decisions service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Too many decisions are in waiting state for operator `{{ "{{" }} $labels.operator {{ "}}" }}`" description: > The cortex scheduling pipeline has a high number of decisions for which @@ -323,15 +323,15 @@ spec: cortex_kpi_state{domain="nova",state!="ready"} != 0 for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: kpis service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/unready annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "KPI `{{ "{{" }} $labels.kpi {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the KPI @@ -342,15 +342,15 @@ spec: expr: cortex_pipeline_state{domain="nova",state!="ready"} != 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: pipelines service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/unready annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Pipeline `{{ "{{" }} $labels.pipeline {{ "}}" }}` is in `{{ "{{" }} $labels.state {{ "}}" }}` state" description: > This may indicate issues with the pipeline @@ -375,15 +375,15 @@ spec: cortex_reservation_state{domain="nova",type="InFlightReservation",state="unknown",reason="InstanceNotFound"} > 10 for: 2h labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: reservations service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/unready annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "In-flight reservation backlog with reason=InstanceNotFound keeps growing" description: > The number of in-flight reservations that have not yet observed @@ -401,15 +401,15 @@ spec: sum by (pipeline, step, intent) (rate(cortex_filter_weigher_pipeline_step_events_total{service="cortex-nova-metrics", event="image_properties_hv_type_undetermined"}[5m])) > 0.1 for: 15m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: scheduling service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/scheduling annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Nova scheduling frequently cannot determine the hypervisor type from image properties" description: > The `filter_image_properties` step in pipeline `{{ "{{" }} $labels.pipeline {{ "}}" }}` @@ -426,15 +426,15 @@ spec: sum by (pipeline, step, hypervisor_type) (rate(cortex_filter_weigher_pipeline_step_events_total{service="cortex-nova-metrics", event="filter_capabilities_unknown_hypervisor_type"}[5m])) > 0.1 for: 15m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: scheduling service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/scheduling annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Nova capabilities filter frequently encounters unknown hypervisor types" description: > The `filter_capabilities` step in pipeline `{{ "{{" }} $labels.pipeline {{ "}}" }}` @@ -448,15 +448,15 @@ spec: expr: sum by (az, hvtype) (increase(cortex_vm_faults{hvtype=~"CH|QEMU",faultmsg=~".*No valid host was found.*",faultmsg!~".*No such host.*"}[5m])) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: scheduling service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/scheduling annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Nova scheduling cannot find valid KVM hosts" description: > Cortex is seeing new faulty vms in `{{ "{{" }} $labels.az {{ "}}" }}` where Nova scheduling @@ -469,15 +469,15 @@ spec: expr: count by(datasource) (cortex_datasource_seconds_until_reconcile{queued="false",domain="nova"}) > 0 for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: datasources service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/datasources annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "New datasource `{{ "{{" }} $labels.datasource {{ "}}" }}` has not reconciled" description: > A new datasource `{{ "{{" }} $labels.datasource {{ "}}" }}` has been added but has not @@ -490,15 +490,15 @@ spec: and on(datasource) cortex_datasource_state{state="ready",domain="nova"} == 1 for: 30m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: datasources service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/datasources annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Existing datasource `{{ "{{" }} $labels.datasource {{ "}}" }}` is lagging behind" description: > An existing datasource `{{ "{{" }} $labels.datasource {{ "}}" }}` has been queued for @@ -512,15 +512,15 @@ spec: / (sum by (controller) (rate(controller_runtime_reconcile_total{service="cortex-nova-metrics"}[5m]))) > 0.1 for: 15m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: controller-errors service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/reconciles annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Controller reconcile error rate >10%" description: > More than 10% of controller reconciles are resulting in errors. This may @@ -534,15 +534,15 @@ spec: / (sum by (controller) (rate(controller_runtime_reconcile_time_seconds_count{service="cortex-nova-metrics"}[5m]))) > {{ .Values.alerts.thresholds.reconcileDurationSeconds }} for: 15m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: controller-duration service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/reconciles annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Controller reconciliation takes longer than ({{ "{{" }} $value | humanizeDuration {{ "}}" }})" description: "Reconcile duration higher than 10m while reconciling {{ "{{" }} $labels.controller {{ "}}" }}" @@ -551,15 +551,15 @@ spec: sum by (name) (workqueue_depth{service="cortex-nova-metrics"}) > 0 for: 60m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: controller-workqueue service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/datasources annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Controller {{ "{{" }} $labels.name {{ "}}" }}'s backlog is not being drained." description: > The workqueue for controller {{ "{{" }} $labels.name {{ "}}" }} has a backlog that is @@ -572,14 +572,14 @@ spec: histogram_quantile(0.9, avg(rate(controller_runtime_webhook_latency_seconds_bucket{service="cortex-nova-metrics"}[5m])) by (webhook, le)) > 0.2 for: 15m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: controller-webhook service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Controller webhook {{ "{{" }} $labels.webhook {{ "}}" }} latency is high" description: > The latency for webhook {{ "{{" }} $labels.webhook {{ "}}" }} is higher than expected (p90 > 200ms). @@ -592,14 +592,14 @@ spec: / (sum by (webhook) (rate(controller_runtime_webhook_requests_total{service="cortex-nova-metrics"}[5m]))) > 0.1 for: 15m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: controller-webhook service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Controller webhook {{ "{{" }} $labels.webhook {{ "}}" }} is experiencing errors" description: > The webhook {{ "{{" }} $labels.webhook {{ "}}" }} has experienced errors in the last 5 minutes. @@ -613,15 +613,15 @@ spec: rate(cortex_committed_resource_info_api_requests_total{service="cortex-nova-metrics", status_code="503"}[5m]) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Committed Resource info API is unavailable" description: > The committed resource info API (Limes LIQUID integration) has been returning @@ -635,15 +635,15 @@ spec: rate(cortex_committed_resource_change_api_requests_total{service="cortex-nova-metrics", status_code=~"5.."}[5m]) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Committed Resource change API HTTP 5xx errors" description: > The committed resource change API (Limes LIQUID integration) is returning @@ -660,15 +660,15 @@ spec: }[5m]) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Committed resource change API returning internal errors" description: > The committed resource change API encountered internal errors processing commitment @@ -685,15 +685,15 @@ spec: and on() sum(rate(cortex_committed_resource_change_api_commitment_changes_total{service="cortex-nova-metrics", result=~"rejected|accepted", dry_run="false"}[1h])) > 0 for: 30m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-performance annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Committed Resource rejection rate too high ({{ "{{" }} $value | humanizePercentage {{ "}}" }})" description: > More than 30% of commitment changes have been rejected over the last hour. @@ -708,14 +708,14 @@ spec: }[10m]) > 0 for: 30m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-api service: cortex severity: info support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Committed resource change API receiving bad requests" description: > The committed resource change API has been receiving requests with invalid parameters @@ -727,15 +727,15 @@ spec: expr: increase(cortex_committed_resource_change_api_timeouts_total{service="cortex-nova-metrics", dry_run="false"}[10m]) > 0 for: 1m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-performance annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Committed Resource change API timeout detected" description: > A commitment change request timed out after the configured deadline. @@ -748,15 +748,15 @@ spec: and on() sum(rate(cortex_committed_resource_change_api_requests_total{service="cortex-nova-metrics", dry_run="false"}[5m])) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-performance annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Committed Resource change API p95 latency >= 10s" description: > The committed resource change API p95 latency has reached or exceeded 10 seconds, @@ -769,15 +769,15 @@ spec: rate(cortex_committed_resource_capacity_api_requests_total{service="cortex-nova-metrics", status_code=~"5.."}[5m]) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Committed Resource capacity API HTTP 5xx errors" description: > The committed resource capacity API (Limes LIQUID integration) is returning @@ -790,15 +790,15 @@ spec: and on(resource, az, unit_size) (cortex_committed_resource_reported_capacity{service="cortex-nova-metrics"} offset 30m > 0) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Committed Resource capacity for {{ "{{" }} $labels.resource {{ "}}" }} in {{ "{{" }} $labels.az {{ "}}" }} dropped to zero" description: > The reported capacity for committed resource {{ "{{" }} $labels.resource {{ "}}" }} in @@ -811,15 +811,15 @@ spec: cortex_committed_resource_capacity_ready{service="cortex-nova-metrics"} == 0 for: 10m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-capacity service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "FlavorGroupCapacity for {{ "{{" }} $labels.flavor_group {{ "}}" }} in {{ "{{" }} $labels.az {{ "}}" }} has been not-ready for >10 minutes" description: > The FlavorGroupCapacity CRD for flavor group {{ "{{" }} $labels.flavor_group {{ "}}" }} @@ -836,15 +836,15 @@ spec: rate(cortex_committed_resource_usage_api_requests_total{service="cortex-nova-metrics", status_code=~"5.."}[5m]) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Committed Resource usage API HTTP 5xx errors" description: > The committed resource usage API (Limes LIQUID integration) is returning @@ -857,15 +857,15 @@ spec: rate(cortex_committed_resource_quota_api_requests_total{service="cortex-nova-metrics", status_code=~"5.."}[5m]) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Committed Resource quota API HTTP 5xx errors" description: > The committed resource quota API (Limes LIQUID integration) is returning @@ -877,14 +877,14 @@ spec: sum by (method, gvk) (increase(cortex_multicluster_cross_cluster_name_conflicts_total{service="cortex-nova-metrics"}[10m])) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: multicluster service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Cross-cluster name conflicts detected for `{{ "{{" }} $labels.gvk {{ "}}" }}`" description: > The multicluster client detected the same resource name for @@ -906,14 +906,14 @@ spec: max by (gvk, host) (cortex_cache_overlay_entries{service="cortex-nova-metrics"}) > {{ .Values.alerts.thresholds.cacheOverlayMaxEntries }} for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: multicluster service: cortex severity: warning support_group: workload-management annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Cache overlay for `{{ "{{" }} $labels.gvk {{ "}}" }}` is not draining" description: > The in-process cache overlay for `{{ "{{" }} $labels.gvk {{ "}}" }}` @@ -946,15 +946,15 @@ spec: ) > 0 for: 15m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-capacity service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Host {{ "{{" }} $labels.compute_host {{ "}}" }} reservation blocks exceed capacity for {{ "{{" }} $labels.resource {{ "}}" }}" description: > The total of running VM allocations and reservation blocks (committed resource + @@ -967,15 +967,15 @@ spec: expr: max by (host, az, resource) (cortex_committed_resource_host_oversubscribed) > 0 for: 15m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-capacity service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Host {{ "{{" }} $labels.host {{ "}}" }} CR slots exceed effective capacity for {{ "{{" }} $labels.resource {{ "}}" }}" description: > Committed resource reservation slots on host {{ "{{" }} $labels.host {{ "}}" }} exceed @@ -985,15 +985,15 @@ spec: - alert: CortexCommittedResourceHostOversubscriptionEvictionsHigh expr: sum by (az) (increase(cortex_committed_resource_host_oversubscribed_evicted_reservations_total{service="cortex-nova-metrics"}[1h])) > 10 labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: committed-resource-capacity service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "High CR slot eviction rate in AZ {{ "{{" }} $labels.az {{ "}}" }}" description: > More than 10 committed resource reservation slots were evicted due to oversubscription @@ -1010,15 +1010,15 @@ spec: # e.g. no eligible hypervisors, exhausted capacity, or scheduling failures. expr: max(min_over_time(cortex_failover_reconciliation_vms_missing_failover{service="cortex-nova-metrics"}[{{ .Values.alerts.thresholds.failoverMissingFor }}])) > 0 labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" +{{- end }} context: failover-reservations service: cortex severity: warning support_group: workload-management playbook: "" annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-status-dashboard" -{{- end }} summary: "Failover reservations missing for VMs" description: > {{ "{{" }} $value | humanize {{ "}}" }} VM(s) never regained their required failover diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index c3fe2b356..28b643566 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -18,11 +18,11 @@ alerts: enabled: true prometheus: openstack # Base URL (no trailing slash) for Perses dashboard links emitted as the - # `persesDashboard` annotation on each alert. The per-alert dashboard slug is + # `persesDashboard` label on each alert. The per-alert dashboard slug is # appended to it. The value is templateable with Prometheus label expressions, # e.g. a region-specific host: # https://perses.{{ $labels.region }}.example.com/projects/observability/dashboards - # Left empty by default so no persesDashboard annotation is emitted. + # Left empty by default so no persesDashboard label is emitted. persesDashboardBaseURL: "" thresholds: # Fraction of each pod's configured container memory limit above which diff --git a/helm/bundles/cortex-placement-shim/templates/alerts.yaml b/helm/bundles/cortex-placement-shim/templates/alerts.yaml index f4ad4c8c6..24cafdcb6 100644 --- a/helm/bundles/cortex-placement-shim/templates/alerts.yaml +++ b/helm/bundles/cortex-placement-shim/templates/alerts.yaml @@ -20,15 +20,15 @@ spec: absent(up{pod=~"cortex-placement-shim-.*"}) for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" +{{- end }} context: liveness service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/shim-down annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" -{{- end }} summary: "Cortex Placement Shim is down" description: > The Cortex Placement Shim is down. Placement API requests that are @@ -48,15 +48,15 @@ spec: avg_over_time(cortex_placement_shim_manager_up{service="cortex-placement-shim-metrics-service"}[15m]) < 0.9 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" +{{- end }} context: liveness service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/shim-down annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" -{{- end }} summary: "Cortex Placement Shim controller-manager is not running" description: > The Cortex Placement Shim controller-manager has been up with a synced @@ -74,15 +74,15 @@ spec: expr: rate(cortex_placement_shim_downstream_request_duration_seconds_count{service="cortex-placement-shim-metrics-service", responsecode=~"4.."}[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" +{{- end }} context: api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/shim-api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" -{{- end }} summary: "Placement Shim downstream HTTP 4xx errors too high" description: > The Placement Shim is responding to client requests with HTTP 4xx @@ -95,15 +95,15 @@ spec: expr: rate(cortex_placement_shim_downstream_request_duration_seconds_count{service="cortex-placement-shim-metrics-service", responsecode=~"5.."}[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" +{{- end }} context: api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/shim-api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" -{{- end }} summary: "Placement Shim downstream HTTP 5xx errors too high" description: > The Placement Shim is responding to client requests with HTTP 5xx @@ -116,15 +116,15 @@ spec: expr: rate(cortex_placement_shim_upstream_request_duration_seconds_count{service="cortex-placement-shim-metrics-service", responsecode=~"5.."}[5m]) > 0.1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" +{{- end }} context: upstream service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/shim-api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" -{{- end }} summary: "Placement Shim upstream HTTP 5xx errors too high" description: > The upstream Placement API is returning 5xx errors to the shim. @@ -139,15 +139,15 @@ spec: and on() sum(rate(cortex_placement_shim_downstream_request_duration_seconds_count{service="cortex-placement-shim-metrics-service"}[5m])) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" +{{- end }} context: api service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/shim-api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" -{{- end }} summary: "Placement Shim downstream latency too high" description: > The Placement Shim downstream request latency (p95) exceeds 10 @@ -162,15 +162,15 @@ spec: and on() sum(rate(cortex_placement_shim_upstream_request_duration_seconds_count{service="cortex-placement-shim-metrics-service"}[5m])) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" +{{- end }} context: upstream service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/shim-api-errors annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" -{{- end }} summary: "Placement Shim upstream latency too high" description: > The upstream Placement API response latency (p95) as seen by the @@ -183,15 +183,15 @@ spec: expr: process_resident_memory_bytes{service="cortex-placement-shim-metrics-service"} > 1500 * 1024 * 1024 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" +{{- end }} context: memory service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/shim-resource-usage annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" -{{- end }} summary: "Placement Shim uses too much memory" description: > The Placement Shim is using more than 1500 MiB of resident memory @@ -203,15 +203,15 @@ spec: expr: rate(process_cpu_seconds_total{service="cortex-placement-shim-metrics-service"}[1m]) > 0.4 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" +{{- end }} context: cpu service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/shim-resource-usage annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" -{{- end }} summary: "Placement Shim uses too much CPU" description: > The Placement Shim is consuming more than 40% of a single CPU core @@ -225,15 +225,15 @@ spec: sum by (method, gvk) (increase(cortex_multicluster_cross_cluster_name_conflicts_total{service="cortex-placement-shim-metrics-service"}[10m])) > 0 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" +{{- end }} context: multicluster service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/shim-multicluster annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" -{{- end }} summary: "Cross-cluster name conflicts detected for `{{ "{{" }} $labels.gvk {{ "}}" }}`" description: > The multicluster client detected the same resource name for @@ -253,15 +253,15 @@ spec: min by (host) (cortex_multicluster_remote_apiserver_reachable{service="cortex-placement-shim-metrics-service"}) < 1 for: 5m labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" +{{- end }} context: multicluster service: cortex severity: warning support_group: workload-management playbook: docs/support/playbook/cortex/alerts/shim-multicluster annotations: -{{- if .Values.alerts.persesDashboardBaseURL }} - persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex-placement-shim-status-dashboard" -{{- end }} summary: "Remote apiserver `{{ "{{" }} $labels.host {{ "}}" }}` is unreachable" description: > The multicluster client cannot reach the remote apiserver at diff --git a/helm/bundles/cortex-placement-shim/values.yaml b/helm/bundles/cortex-placement-shim/values.yaml index df1db661f..09c36554b 100644 --- a/helm/bundles/cortex-placement-shim/values.yaml +++ b/helm/bundles/cortex-placement-shim/values.yaml @@ -18,11 +18,11 @@ alerts: enabled: true prometheus: openstack # Base URL (no trailing slash) for Perses dashboard links emitted as the - # `persesDashboard` annotation on each alert. The per-alert dashboard slug is + # `persesDashboard` label on each alert. The per-alert dashboard slug is # appended to it. The value is templateable with Prometheus label expressions, # e.g. a region-specific host: # https://perses.{{ $labels.region }}.example.com/projects/observability/dashboards - # Left empty by default so no persesDashboard annotation is emitted. + # Left empty by default so no persesDashboard label is emitted. persesDashboardBaseURL: "" cortex-shim: From bf33b8359f48005a841b6a5620d9da1ddb10365b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:12:21 +0200 Subject: [PATCH 19/20] bump app version [skip ci] (#1387) bump app version [skip ci] ``` bumped cortex: sha-c6824705 -> sha-b4cf3be3 noop cortex-shim (sha-97182126) noop cortex-postgres (sha-dbe244dd) ``` Signed-off-by: SoWieMarkus <44964229+SoWieMarkus@users.noreply.github.com> Co-authored-by: SoWieMarkus <44964229+SoWieMarkus@users.noreply.github.com> --- helm/library/cortex/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index bb49b2b0b..5abcf30e9 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -3,6 +3,6 @@ name: cortex description: A Helm chart to distribute cortex. type: application version: 0.6.8 -appVersion: "sha-c6824705" +appVersion: "sha-b4cf3be3" icon: "https://example.com/icon.png" dependencies: [] From bbc24e032b756d8fcb1be592b88c206452d6f5b5 Mon Sep 17 00:00:00 2001 From: "cortex-ai-agents[bot]" <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:50:26 +0200 Subject: [PATCH 20/20] Release cortex 0.7.0 (#1386) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Release prep for #1385: changelog entry and helm chart version bumps. Merge this before merging #1385. ## Changes - Updated CHANGELOG.md with release notes for cortex 0.7.0 - Bumped helm chart versions across 10 Chart.yaml files in `helm/library` and `helm/bundles` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 --- CHANGELOG.md | 60 +++++++++++++++++++ helm/bundles/cortex-cinder/Chart.yaml | 8 +-- helm/bundles/cortex-crds/Chart.yaml | 4 +- helm/bundles/cortex-ironcore/Chart.yaml | 4 +- helm/bundles/cortex-manila/Chart.yaml | 8 +-- helm/bundles/cortex-nova/Chart.yaml | 8 +-- helm/bundles/cortex-placement-shim/Chart.yaml | 4 +- helm/bundles/cortex-pods/Chart.yaml | 4 +- helm/library/cortex-postgres/Chart.yaml | 2 +- helm/library/cortex-shim/Chart.yaml | 2 +- helm/library/cortex/Chart.yaml | 2 +- 11 files changed, 83 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 992f33c43..bfddd6931 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,65 @@ # Changelog +## cortex v0.7.0 (2026-10-19) + +### ⚠️ Breaking Changes + +- **History CRD removed:** The `cortex.cloud_histories` CRD and all associated controllers/tasks have been deleted. Any tooling relying on `kubectl get histories` or the history client library must migrate to `spec.cortex.explanation` on the `VirtualMachine` CRD. +- **CRD schema changes:** `Decision` and `VirtualMachine` CRDs updated with new `explanation` and `intent` fields. + +### Features + +- **Improve virtual machine creation logging** — Replaced the opaque boolean `createVM` gate with a `switch` that logs a specific skip reason (including `instanceUUID`) for each unmet precondition, plus logs successful VM create/update events for end-to-end traceability. +- **Populate cortex intent and explanation on VM spec** ([#1373](https://github.com/cobaltcore-dev/cortex/pull/1373)) — Bumped `vm-operator` to v0.2.8 and writes the new `Explanation` and `Intent` fields into the `VirtualMachine` CRD spec during scheduling, replacing the now-removed `History` CRD as the explanation surface. +- **Prune aged committed-resource reservations** ([#1382](https://github.com/cobaltcore-dev/cortex/pull/1382)) — The syncer now detects and deletes stale CR-backed Reservations that exceed a configurable max age (capped per run), so the reconciler can rebuild fresh slots rather than accumulating stale ones. + +### Bug Fixes + +- **Fix broken feature gate configuration in nova bundle** — Feature gate keys (`committedResourceTracking`, `virtualMachineCreation`, `virtualMachineCreationMandatory`) were incorrectly nested under a `featureGates:` block; moved to the correct nesting level. +- **CRs confirm superseded/expired CRs correctly within CR transactions** ([#1372](https://github.com/cobaltcore-dev/cortex/pull/1372)) — Superseded/expired CR transitions are no longer added to the poll-watch list, eliminating `CortexNovaCommittedResourceTimeoutsTooHigh` alerts. Also adds a generation-advance hang guard and wraps reservation deletes in `IgnoreNotFound`. +- **Fix scheduling intent set to Unknown always** ([#1374](https://github.com/cobaltcore-dev/cortex/pull/1374)) — Fixed a bug that caused VM resources to never be generated because intent was always `Unknown`. +- **Default scheduling intent to Create for Nova API** ([#1376](https://github.com/cobaltcore-dev/cortex/pull/1376)) — When intent cannot be determined, it now defaults to `CreateIntent` instead of `Unknown`; removes redundant intent derivation that overrode the fallback. +- **Fix VM status condition being wiped on create path** — Snapshots conditions before `Create` and restores them before the status sub-resource update so the `Scheduled` condition is preserved. +- **Use decision intent in VM handler instead of re-deriving from request** ([#1379](https://github.com/cobaltcore-dev/cortex/pull/1379)) — Reads intent from `decision.Spec.Intent` (which already incorporates the API's fallback logic) rather than re-calling `GetIntent()` on the request, fixing initial VM create failures. +- **Change execution order** ([#1384](https://github.com/cobaltcore-dev/cortex/pull/1384)) — Corrected execution ordering in the reservations path. + +### Chores & Refactoring + +- **Remove History CRD** ([#1375](https://github.com/cobaltcore-dev/cortex/pull/1375)) — Deleted the `History` custom resource definition, the history client, and all history cleanup tasks across Nova, Cinder, and Manila schedulers. Explanation data is now embedded directly on the `VirtualMachine` CRD. +- **Add edge cases to explanation** ([#1383](https://github.com/cobaltcore-dev/cortex/pull/1383)) — Hardened the plain-text explanation generator with additional edge-case handling. + +### Documentation + +- **Correct Decision CRD section** ([#1381](https://github.com/cobaltcore-dev/cortex/pull/1381)) — Clarified that `Decision` is a transient in-memory CRD (not persisted to etcd); replaced the misleading `kubectl get decisions` example with `kubectl get virtualmachines` showing `spec.cortex.*` fields. + +### Dependency Updates + +| Dependency | Old | New | +|---|---|---| +| `vm-operator` | v0.1.2-pre | v0.2.9 | +| `kube-prometheus-stack` | 92.7.0 | 92.10.1 | +| `controller-tools` | v0.22.0 | v0.23.0 | +| `golangci-lint` | v2.14.0 | v2.15.0 | + +### Package Versions + +| Package | Version | +|---|---| +| cortex | 0.7.0 | +| cortex-postgres | 0.6.28 | +| cortex-shim | 0.1.33 | +| bundles | 0.0.107 | +| cortex-placement-shim | 0.1.33 | + +### Operational Notes + +- **Alert resolution:** This release fixes the `CortexNovaCommittedResourceTimeoutsTooHigh` alert by no longer polling superseded/expired CR transitions. +- **Stale reservation pruning:** CR-backed Reservations are now automatically pruned past a configurable max age — verify the default threshold is appropriate for your deployment. + +**Contributors:** @PhilippMatthes, @mblos, @SoWieMarkus, @cortex-ai-agents[bot], @renovate[bot], @github-actions[bot] + +**PR:** [#1385](https://github.com/cobaltcore-dev/cortex/pull/1385) | **Commits:** 19 | **Files changed:** 79 | **Test coverage:** 71.4% + ## Release 2026-10-15 — PR [#1367](https://github.com/cobaltcore-dev/cortex/pull/1367) ### Bumped Versions diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index 0ea91b342..dde023d8c 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.106 +version: 0.0.107 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.27 + version: 0.6.28 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.8 + version: 0.7.0 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.8 + version: 0.7.0 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index f7e4d3cdb..74b9f46cf 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.106 +version: 0.0.107 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.8 + version: 0.7.0 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index fdae16a8a..79e796cb1 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.106 +version: 0.0.107 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.8 + version: 0.7.0 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index 82cace9a7..cbfd735ad 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.106 +version: 0.0.107 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.27 + version: 0.6.28 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.8 + version: 0.7.0 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.8 + version: 0.7.0 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index abe0d30f8..ab91645f7 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.106 +version: 0.0.107 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.27 + version: 0.6.28 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.8 + version: 0.7.0 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.8 + version: 0.7.0 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index 862eda447..c0d19b9cb 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.32 +version: 0.1.33 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.32 + version: 0.1.33 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index 2fa7941ac..f68b5d008 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.106 +version: 0.0.107 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.8 + version: 0.7.0 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index cfbed9775..913ecbd1a 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -5,5 +5,5 @@ apiVersion: v2 name: cortex-postgres description: Postgres setup for Cortex. type: application -version: 0.6.27 +version: 0.6.28 appVersion: "sha-dbe244dd" diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index 8ae6fd5cb..abc48c0a6 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.32 +version: 0.1.33 appVersion: "sha-97182126" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index 5abcf30e9..2b81bbc10 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.6.8 +version: 0.7.0 appVersion: "sha-b4cf3be3" icon: "https://example.com/icon.png" dependencies: []