diff --git a/deploy/helm/helm-reval/values.yaml b/deploy/helm/helm-reval/values.yaml index 55ed54d71..09a10baed 100644 --- a/deploy/helm/helm-reval/values.yaml +++ b/deploy/helm/helm-reval/values.yaml @@ -83,9 +83,9 @@ reval: enabled: false serviceConfig: - skipValidateObjects: false - skipValidateImages: false - skipSanitizeObjectMetadata: false + skipValidateObjects: true + skipValidateImages: true + skipSanitizeObjectMetadata: true preserveLabels: [] preserveAnnotations: [] diff --git a/deploy/stacks/nvcf-compute-plane/Makefile b/deploy/stacks/nvcf-compute-plane/Makefile index 1ede68045..9c90ee93d 100644 --- a/deploy/stacks/nvcf-compute-plane/Makefile +++ b/deploy/stacks/nvcf-compute-plane/Makefile @@ -130,6 +130,9 @@ dist: clean-dist # Copy helmfile.d files @cp -r "$(MAKEFILE_DIR)/helmfile.d" "$(DIST_DIR)/" + # Copy local charts referenced by helmfile.d releases via relative paths + @cp -r "$(MAKEFILE_DIR)/charts" "$(DIST_DIR)/" + # Copy and transform README (remove internal references) @sed -e 's|nvcf-ncp-internal/ncp-dev|your-org/your-team|g' \ -e 's|0651155215864979/ncp-dev|YOUR_ORG/YOUR_TEAM|g' \ diff --git a/deploy/stacks/nvcf-compute-plane/README.md b/deploy/stacks/nvcf-compute-plane/README.md index 2ac979a0d..811a48cc3 100644 --- a/deploy/stacks/nvcf-compute-plane/README.md +++ b/deploy/stacks/nvcf-compute-plane/README.md @@ -89,13 +89,20 @@ make install CLUSTER_NAME=... # downloads helmfile v1.1.9 + helm v3.15.4 on fi ## Optional Add-ons -KAI Scheduler, Grove (topology-aware scheduling), and Dynamo (inference framework -scheduling) are disabled by default. Grove and Dynamo require KAI Scheduler -(release and namespace `kai-scheduler`). Enable per-environment in -`environments/.yaml`: +The stack can install KAI Scheduler, Grove, and Dynamo. KAI provides resource +allocation and gang placement, Grove orchestrates related Pod groups, and +Dynamo describes inference services that Grove manages. GPU clique +topology-aware scheduling connects those workloads to the cluster's NVLink +topology. + +All add-ons are disabled by default. Grove requires KAI Scheduler, and Dynamo +requires Grove. Topology-aware scheduling also requires KAI Scheduler. Enable +the complete stack in `environments/.yaml`: ```yaml addons: + topologyAwareScheduling: + enabled: true kaiScheduler: enabled: true groveOperator: @@ -108,8 +115,23 @@ Override KAI component resources under `addons.kaiScheduler..resource (for example `addons.kaiScheduler.scheduler.resources.requests.memory`). Defaults are set in `helmfile.d/01-dependencies.yaml.gotmpl`. -For a standalone KAI install outside this stack, follow the -[KAI Scheduler guide](https://docs.nvidia.com/cloud-functions/current/latest/cluster-management/kai-scheduler.html). +`addons.topologyAwareScheduling` installs cluster-scoped KAI `Topology` +resources from `topologyAwareScheduling.topologies` when KAI is enabled. +When Grove is also enabled, the same toggle sets Grove +`topologyAwareScheduling.enabled=true` and installs one +`ClusterTopologyBinding` per KAI `Topology`. The default topology labels are +`nvidia.com/gpu.clique` and `kubernetes.io/hostname`, in that order. + +Enabling `addons.kaiScheduler.enabled` or `addons.dynamoOperator.enabled` also +adds the matching NVCA feature gate. Enabling KAI, Grove, or Dynamo permits +their workload resource types in the NVCA validation policy. + +See [Gang Scheduling](../../../docs/user/cluster-management/gang-scheduling.md) +for atomic workload placement and +[Topology-Aware Scheduling](../../../docs/user/cluster-management/topology-aware-scheduling.md) +for GPU clique placement. See +[KAI Scheduler](../../../docs/user/cluster-management/kai-scheduler.md) for +queue configuration and standalone installation. ## Multi-Cluster Example diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml b/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/Chart.yaml similarity index 74% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml rename to deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/Chart.yaml index bc718adce..472ea8790 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml +++ b/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/Chart.yaml @@ -1,5 +1,3 @@ ---- -# Source: helm-nvca-operator/templates/rbac_allowed_extra_types.yaml # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -14,3 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +apiVersion: v2 +name: nvcf-cluster-topology +description: >- + Cluster-scoped KAI Topology and Grove ClusterTopologyBinding resources for + NVCF MNNVL topology-aware scheduling +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/_helpers.tpl b/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/_helpers.tpl new file mode 100644 index 000000000..217095e00 --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/_helpers.tpl @@ -0,0 +1,40 @@ +{{/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "nvcf-cluster-topology.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "nvcf-cluster-topology.labels" -}} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +app.kubernetes.io/name: {{ include "nvcf-cluster-topology.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end -}} + +{{/* +Derive a Grove domain name from a KAI nodeLabel: last path element with '.' removed. +Example: nvidia.com/gpu.clique -> gpuclique; kubernetes.io/hostname -> hostname +*/}} +{{- define "nvcf-cluster-topology.groveDomain" -}} +{{- $nodeLabel := . | required "nvcf-cluster-topology: every level must set nodeLabel" -}} +{{- $nodeLabel | splitList "/" | last | replace "." "" -}} +{{- end -}} diff --git a/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/cluster-topology-binding.yaml b/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/cluster-topology-binding.yaml new file mode 100644 index 000000000..5cc9b1e31 --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/cluster-topology-binding.yaml @@ -0,0 +1,62 @@ +{{/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.groveOperator.enabled }} +{{- $topologies := .Values.topologies | default list }} +{{- if empty $topologies }} +{{- fail "nvcf-cluster-topology: topologies must contain at least one entry when groveOperator.enabled is true" }} +{{- end }} +{{- $schedulerName := .Values.groveOperator.topologyBindingSchedulerName | default "kai-scheduler" }} +{{- range $i, $topo := $topologies }} +{{- $levels := $topo.levels | default list }} +{{- if empty $levels }} +{{- fail (printf "nvcf-cluster-topology: topology %q must set at least one level" ($topo.name | default "")) }} +{{- end }} +{{- $topoName := $topo.name | required "nvcf-cluster-topology: every topology must set name" }} +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: grove.io/v1alpha1 +kind: ClusterTopologyBinding +metadata: + name: {{ printf "%s-binding" $topoName }} + labels: + {{- include "nvcf-cluster-topology.labels" $ | nindent 4 }} +spec: + levels: + {{- range $levels }} + - domain: {{ include "nvcf-cluster-topology.groveDomain" .nodeLabel | quote }} + key: {{ .nodeLabel | required "nvcf-cluster-topology: every level must set nodeLabel" | quote }} + {{- end }} + schedulerTopologyBindings: + - schedulerName: {{ $schedulerName | quote }} + topologyReference: {{ $topoName | quote }} +{{- end }} +{{- end }} diff --git a/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/topology.yaml b/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/topology.yaml new file mode 100644 index 000000000..588674d98 --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/templates/topology.yaml @@ -0,0 +1,56 @@ +{{/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.kaiScheduler.enabled }} +{{- $topologies := .Values.topologies | default list }} +{{- if empty $topologies }} +{{- fail "nvcf-cluster-topology: topologies must contain at least one entry when kaiScheduler.enabled is true" }} +{{- end }} +{{- range $i, $topo := $topologies }} +{{- $levels := $topo.levels | default list }} +{{- if empty $levels }} +{{- fail (printf "nvcf-cluster-topology: topology %q must set at least one level" ($topo.name | default "")) }} +{{- end }} +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kai.scheduler/v1alpha1 +kind: Topology +metadata: + name: {{ $topo.name | required "nvcf-cluster-topology: every topology must set name" }} + labels: + {{- include "nvcf-cluster-topology.labels" $ | nindent 4 }} +spec: + levels: + {{- range $levels }} + - nodeLabel: {{ .nodeLabel | required "nvcf-cluster-topology: every level must set nodeLabel" | quote }} + {{- end }} +{{- end }} +{{- end }} diff --git a/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/values.yaml b/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/values.yaml new file mode 100644 index 000000000..58c3f63da --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/charts/nvcf-cluster-topology/values.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cluster-scoped Topology resources to install when kaiScheduler.enabled is +# true. Workloads reference a Topology by name through the +# `kai.scheduler/topology` annotation. +# +# Each entry needs: +# name: Topology resource name +# levels: node labels ordered widest domain first, narrowest last +# +# When groveOperator.enabled is true, each topologies entry also yields a +# ClusterTopologyBinding named -binding. Grove level domain is the last +# path element of nodeLabel with '.' removed; key is nodeLabel. +topologies: +- name: nvcf-mnnvl-topology + levels: + - nodeLabel: nvidia.com/gpu.clique + - nodeLabel: kubernetes.io/hostname + +# When true, render kai.scheduler/v1alpha1 Topology CRs from topologies. +kaiScheduler: + enabled: false + +# When true, render a grove.io/v1alpha1 ClusterTopologyBinding per topologies +# entry, derived from that entry's levels. +# topologyBindingSchedulerName sets schedulerTopologyBindings[].schedulerName. +groveOperator: + enabled: false + topologyBindingSchedulerName: kai-scheduler + +commonLabels: {} diff --git a/deploy/stacks/nvcf-compute-plane/environments/base.yaml b/deploy/stacks/nvcf-compute-plane/environments/base.yaml index b987ee657..5d38f46db 100644 --- a/deploy/stacks/nvcf-compute-plane/environments/base.yaml +++ b/deploy/stacks/nvcf-compute-plane/environments/base.yaml @@ -72,18 +72,39 @@ observability: # ============================================================================= addons: + # MNNVL topology-aware scheduling (disabled by default). Enable on + # NVLink-optimized clusters so multi-node functions can request clique-level + # gang placement. When true and kaiScheduler.enabled, installs KAI Topology + # CRs from topologies below. When true and groveOperator.enabled, also sets + # Grove topologyAwareScheduling.enabled and installs a ClusterTopologyBinding + # that references the KAI Topology. Requires addons.kaiScheduler.enabled. + # Function owners opt in per workload with the `kai.scheduler/topology` + # annotation naming one of the entries below. + topologyAwareScheduling: + enabled: false + topologies: + - name: nvcf-mnnvl-topology + # Ordered widest domain first, narrowest last. + levels: + - nodeLabel: nvidia.com/gpu.clique + - nodeLabel: kubernetes.io/hostname + # KAI Scheduler (disabled by default). Required when enabling Grove or Dynamo. + # When enabled, NVCA also gets the KAIScheduler feature gate. # Per-component resource overrides: addons.kaiScheduler..resources # (requests/limits). Defaults live in helmfile.d/01-dependencies.yaml.gotmpl. kaiScheduler: enabled: false - # Grove topology-aware scheduling operator (disabled by default) + # Grove topology-aware scheduling operator (disabled by default). + # topologyAwareScheduling.enabled is driven by addons.topologyAwareScheduling + # when that top-level toggle is on. Override network settings explicitly + # (for example network.autoMNNVLEnabled); they are not set by the topology toggle. groveOperator: enabled: false - topologyAwareScheduling: {} network: {} - # Dynamo inference framework scheduling operator (disabled by default) + # Dynamo inference framework scheduling operator (disabled by default). + # When enabled, NVCA also gets the DynamoOperatorSupport feature gate. dynamoOperator: enabled: false diff --git a/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl b/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl index 416159a10..8b3baa242 100644 --- a/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl +++ b/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl @@ -1,9 +1,13 @@ # NVCF Compute Plane Releases # -# Optional releases — installed once per GPU cluster: -# kai-scheduler — optional; addons.kaiScheduler.enabled (required by grove/dynamo) -# grove-operator — optional; topology-aware scheduling (addons.groveOperator.enabled) -# dynamo-operator — optional; Dynamo inference framework scheduling (addons.dynamoOperator.enabled) +# Optional releases, installed once per GPU cluster: +# kai-scheduler: addons.kaiScheduler.enabled (required by grove/dynamo) +# nvcf-kai-topology: KAI Topology CRs +# (topologyAwareScheduling + kaiScheduler) +# grove-operator: topology-aware scheduling (addons.groveOperator.enabled) +# nvcf-grove-topology: Grove ClusterTopologyBinding CRs +# (topologyAwareScheduling + groveOperator; needs KAI Topology) +# dynamo-operator: Dynamo inference framework scheduling (addons.dynamoOperator.enabled) # # CLUSTER_NAME is required and scopes the registration values handoff file. # Run `make register-cluster CLUSTER_NAME=` before `make install`. @@ -228,6 +232,43 @@ releases: wait: true waitForJobs: true + - name: nvcf-kai-topology + # Cluster-scoped KAI Scheduler Topology resources (optional; disabled by + # default). Workloads opt in through the `kai.scheduler/topology` + # annotation so KAI places every replica of a StatefulSet inside a single + # clique instead of binding pods one at a time. + # + # Installed when addons.topologyAwareScheduling.enabled and + # addons.kaiScheduler.enabled are both true. Configure the Topology list + # under addons.topologyAwareScheduling.topologies. + # + # Requires the kai-scheduler release (and KAI v0.12.0 or later) for the + # native `kai.scheduler/v1alpha1` Topology CRD. + # + # Resources are cluster-scoped; the release lives in the nvca-operator + # namespace so lifecycle stays with NVCA-owned resources. + {{- $topoEnabled := dig "topologyAwareScheduling" "enabled" false .Values.addons }} + {{- $kaiEnabled := dig "kaiScheduler" "enabled" false .Values.addons }} + {{- if and $topoEnabled (not $kaiEnabled) }} + {{- fail "addons.topologyAwareScheduling.enabled requires addons.kaiScheduler.enabled" }} + {{- end }} + installed: {{ and $topoEnabled $kaiEnabled }} + chart: ../charts/nvcf-cluster-topology + namespace: nvca-operator + needs: + - kai-scheduler/kai-scheduler + values: + - topologies: + {{- toYaml (dig "topologyAwareScheduling" "topologies" list .Values.addons) | nindent 10 }} + kaiScheduler: + enabled: true + groveOperator: + enabled: false + labels: + release-group: workers + wait: true + waitForJobs: true + - name: grove-operator # Topology-aware scheduling operator (optional; disabled by default). # Enable per-environment: addons.groveOperator.enabled: true @@ -274,9 +315,9 @@ releases: concurrentSyncs: 3 podCliqueScalingGroup: concurrentSyncs: 3 - {{- with .Values.addons.groveOperator.topologyAwareScheduling }} + {{- if dig "topologyAwareScheduling" "enabled" false .Values.addons }} topologyAwareScheduling: - {{- toYaml . | nindent 12 }} + enabled: true {{- end }} {{- with .Values.addons.groveOperator.network }} network: @@ -305,6 +346,35 @@ releases: wait: true waitForJobs: true + - name: nvcf-grove-topology + # Cluster-scoped Grove ClusterTopologyBinding resources (optional). + # Installed when addons.topologyAwareScheduling.enabled and + # addons.groveOperator.enabled are both true. Bindings are derived from + # addons.topologyAwareScheduling.topologies and reference the KAI Topology + # CRs created by nvcf-kai-topology. + # + # Waits on grove-operator (ClusterTopologyBinding CRD) and + # nvcf-kai-topology (Topology CRs referenced by topologyReference). + {{- $topoEnabled := dig "topologyAwareScheduling" "enabled" false .Values.addons }} + {{- $groveEnabled := dig "groveOperator" "enabled" false .Values.addons }} + installed: {{ and $topoEnabled $groveEnabled }} + chart: ../charts/nvcf-cluster-topology + namespace: nvca-operator + needs: + - grove-system/grove-operator + - nvca-operator/nvcf-kai-topology + values: + - topologies: + {{- toYaml (dig "topologyAwareScheduling" "topologies" list .Values.addons) | nindent 10 }} + kaiScheduler: + enabled: false + groveOperator: + enabled: true + labels: + release-group: workers + wait: true + waitForJobs: true + - name: dynamo-operator # Dynamo inference framework scheduling operator (optional; disabled by default). # Enable per-environment: addons.dynamoOperator.enabled: true diff --git a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl index 1b5f457d1..6e1f2b6d7 100644 --- a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl +++ b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl @@ -19,8 +19,14 @@ environments: Every gate in the default set below is enabled unless it is already present or explicitly disabled by including "-" in the input. This keeps the defaults intact even when the operator sets a non-empty featureGateValues - list, which would otherwise replace the chart default entirely. Input (.) is - the raw list; output is the normalized list as YAML. + list, which would otherwise replace the chart default entirely. + + Input is a dict: + values: operator-supplied featureGateValues list + byooEnabled: also enable BYOObservability + kaiSchedulerEnabled: also enable KAIScheduler (addons.kaiScheduler.enabled) + dynamoOperatorEnabled: also enable DynamoOperatorSupport (addons.dynamoOperator.enabled) + Output is the normalized list as YAML. */}} {{- define "nvca-operator.featureGateValues" -}} {{- $featureGateValues := dig "values" list . | default list -}} @@ -35,6 +41,12 @@ environments: {{- if dig "byooEnabled" false . -}} {{- $defaultFeatureGates = append $defaultFeatureGates "BYOObservability" -}} {{- end -}} +{{- if dig "kaiSchedulerEnabled" false . -}} +{{- $defaultFeatureGates = append $defaultFeatureGates "KAIScheduler" -}} +{{- end -}} +{{- if dig "dynamoOperatorEnabled" false . -}} +{{- $defaultFeatureGates = append $defaultFeatureGates "DynamoOperatorSupport" -}} +{{- end -}} {{- range $gate := $defaultFeatureGates -}} {{- if not (or (has $gate $featureGateValues) (has (printf "-%s" $gate) $featureGateValues)) -}} {{- $featureGateValues = append $featureGateValues $gate -}} @@ -76,6 +88,65 @@ helmDefaults: {{- $agentMergeConfig = mergeOverwrite $agentMergeConfig ($environmentAgentMergeConfig | fromYaml | default dict) }} {{- end }} +{{- $selfManaged := dig "nvcaOperator" "selfManaged" dict .Values.global }} +{{- $featureGateValues := dig "featureGateValues" list $selfManaged }} +{{- $kaiSchedulerEnabled := dig "addons" "kaiScheduler" "enabled" false .Values }} +{{- $dynamoOperatorEnabled := dig "addons" "dynamoOperator" "enabled" false .Values }} + +{{- /* + Extra Kubernetes types owned by KAI Scheduler, Grove, and Dynamo. NVCA needs + these in its validation policy to render, admit, and garbage collect the + objects those controllers create for a Helm function, and the nvca-operator + chart derives the matching ClusterRole from the same list. Adding them here + means an operator does not have to restate the whole policy just to run + multi-node or Dynamo workloads. Entries supplied through registration or + environment values always win over these defaults. +*/}} +{{- $extraTypes := list }} +{{- if or $kaiSchedulerEnabled (has "KAIScheduler" $featureGateValues) }} +{{- $extraTypes = append $extraTypes (dict "group" "scheduling.run.ai" "version" "v2alpha2" "kind" "PodGroup" "resource" "podgroups") }} +{{- end }} +{{- if dig "addons" "groveOperator" "enabled" false .Values }} +{{- $extraTypes = concat $extraTypes (list + (dict "group" "grove.io" "version" "v1alpha1" "kind" "PodCliqueSet" "resource" "podcliquesets") + (dict "group" "grove.io" "version" "v1alpha1" "kind" "PodClique" "resource" "podcliques") + (dict "group" "grove.io" "version" "v1alpha1" "kind" "PodCliqueScalingGroup" "resource" "podcliquescalinggroups") + (dict "group" "scheduler.grove.io" "version" "v1alpha1" "kind" "PodGang" "resource" "podgangs") + ) }} +{{- end }} +{{- if or $dynamoOperatorEnabled (has "DynamoOperatorSupport" $featureGateValues) }} +{{- $extraTypes = concat $extraTypes (list + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoCheckpoint" "resource" "dynamocheckpoints") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoComponentDeployment" "resource" "dynamocomponentdeployments") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoGraphDeployment" "resource" "dynamographdeployments") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoGraphDeploymentRequest" "resource" "dynamographdeploymentrequests") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoGraphDeploymentScalingAdapter" "resource" "dynamographdeploymentscalingadapters") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoModel" "resource" "dynamomodels") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoWorkerMetadata" "resource" "dynamoworkermetadatas") + ) }} +{{- end }} +{{- if $extraTypes }} +{{- $validationPolicy := dig "cluster" "validationPolicy" dict $agentMergeConfig }} +{{- $allowedTypes := dig "allowedExtraKubernetesTypes" list $validationPolicy }} +{{- $seen := dict }} +{{- range $allowedTypes }} +{{- $seen = set $seen (printf "%s/%s/%s" .group .version .kind) true }} +{{- end }} +{{- range $extraTypes }} +{{- $key := printf "%s/%s/%s" .group .version .kind }} +{{- if not (hasKey $seen $key) }} +{{- $allowedTypes = append $allowedTypes . }} +{{- end }} +{{- end }} +{{- $validationPolicy = set $validationPolicy "allowedExtraKubernetesTypes" $allowedTypes }} +{{- /* ReVal only accepts "Default" or "Unrestricted"; an unset name is rejected. */}} +{{- if not (dig "name" "" $validationPolicy) }} +{{- $validationPolicy = set $validationPolicy "name" "Default" }} +{{- end }} +{{- $cluster := set (dig "cluster" dict $agentMergeConfig) "validationPolicy" $validationPolicy }} +{{- $agentMergeConfig = set $agentMergeConfig "cluster" $cluster }} +{{- end }} + releases: - name: nvca-operator @@ -137,8 +208,10 @@ releases: {{- end }} featureGateValues: {{- include "nvca-operator.featureGateValues" (dict - "values" (dig "featureGateValues" list $selfManaged) - "byooEnabled" $computeObservabilityEnabled) | nindent 10 }} + "values" $featureGateValues + "byooEnabled" $computeObservabilityEnabled + "kaiSchedulerEnabled" $kaiSchedulerEnabled + "dynamoOperatorEnabled" $dynamoOperatorEnabled) | nindent 10 }} otelCollector: enabled: {{ $otelCollectorEnabled }} {{- with dig "imageRepository" "" $otelCollector }} diff --git a/deploy/stacks/nvcf-compute-plane/testdata/environments/local.yaml b/deploy/stacks/nvcf-compute-plane/testdata/environments/local.yaml index f5ecdef06..669274cda 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/environments/local.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/environments/local.yaml @@ -18,9 +18,24 @@ global: natsURL: nats://nats.nats-system.svc.cluster.local:4222 addons: + topologyAwareScheduling: + enabled: true kaiScheduler: enabled: true groveOperator: enabled: true dynamoOperator: enabled: true + +# Operator-supplied validation policy name is kept. Addon defaults (KAI, Grove, +# Dynamo) are appended and deduplicated against this list. +agentConfig: + mergeConfig: | + cluster: + validationPolicy: + name: Unrestricted + allowedExtraKubernetesTypes: + - group: foo.com + version: v1 + kind: FooType + resource: footypes diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/component-serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/component-serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/component-serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/component-serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/deployment.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/deployment.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/deployment.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/deployment.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/epp.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/epp.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/epp.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/epp.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-preflight.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-preflight.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-preflight.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-preflight.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/leader-election-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/leader-election-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/leader-election-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/leader-election-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/manager-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/manager-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/manager-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/manager-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-auth-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-auth-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-auth-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-auth-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-service.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-service.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-service.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-service.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-config.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-config.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-config.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-config.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-servicemonitor.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-servicemonitor.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-servicemonitor.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-servicemonitor.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/planner.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/planner.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/planner.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/planner.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/profiling-job-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/profiling-job-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/profiling-job-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/profiling-job-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/prometheus.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/prometheus.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/prometheus.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/prometheus.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/regcred-secret.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/regcred-secret.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/regcred-secret.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/regcred-secret.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/validate-values.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/validate-values.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/validate-values.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/validate-values.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-certificates.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-certificates.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-certificates.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-certificates.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-configuration.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-configuration.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-configuration.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-configuration.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-service.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-service.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-service.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-service.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/config-map.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/config-map.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/config-map.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/config-map.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/headless-service.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/headless-service.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/headless-service.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/headless-service.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/pod-disruption-budget.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/pod-disruption-budget.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/pod-disruption-budget.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/pod-disruption-budget.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/service.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/service.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/service.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/service.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/stateful-set.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/stateful-set.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/stateful-set.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/stateful-set.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/templates/kai.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/templates/kai.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/templates/kai.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/templates/kai.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/templates/validate-values.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/templates/validate-values.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/templates/validate-values.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/templates/validate-values.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clusterrole.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clusterrole.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clusterrole.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clusterrole.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clusterrolebinding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clusterrolebinding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clusterrolebinding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clusterrolebinding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clustertopology-validating-webhook-config.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clustertopology-validating-webhook-config.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clustertopology-validating-webhook-config.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clustertopology-validating-webhook-config.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/configmap-operator.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/configmap-operator.yaml similarity index 96% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/configmap-operator.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/configmap-operator.yaml index 645383018..2273541de 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/configmap-operator.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/configmap-operator.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: grove-operator-cm-b77904eb + name: grove-operator-cm-58969b77 namespace: grove-system labels: chart: "grove-charts-v0.1.0-alpha.9" @@ -55,6 +55,6 @@ data: logLevel: info logFormat: json topologyAwareScheduling: - enabled: false + enabled: true network: autoMNNVLEnabled: false diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/deployment.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/deployment.yaml similarity index 98% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/deployment.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/deployment.yaml index ad134cd3e..f779a8068 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/deployment.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/deployment.yaml @@ -106,7 +106,7 @@ spec: apiVersion: v1 - name: operator-config configMap: - name: grove-operator-cm-b77904eb + name: grove-operator-cm-58969b77 - name: grove-webhook-server-cert # Mount the Secret volume containing webhook certificates secret: diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/leaderelection-role.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/leaderelection-role.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/leaderelection-role.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/leaderelection-role.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/leaderelection-rolebinding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/leaderelection-rolebinding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/leaderelection-rolebinding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/leaderelection-rolebinding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/pcs-defaulting-webhook-config.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/pcs-defaulting-webhook-config.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/pcs-defaulting-webhook-config.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/pcs-defaulting-webhook-config.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/pcs-validating-webhook-config.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/pcs-validating-webhook-config.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/pcs-validating-webhook-config.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/pcs-validating-webhook-config.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/service.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/service.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/service.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/service.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/default-queue.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/default-queue.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/default-queue.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/default-queue.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/default-shard.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/default-shard.yaml similarity index 99% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/default-shard.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/default-shard.yaml index da1ddad4e..b599090d6 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/default-shard.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/default-shard.yaml @@ -2,7 +2,6 @@ # Source: kai-scheduler/templates/default-shard.yaml # Copyright 2025 NVIDIA CORPORATION # SPDX-License-Identifier: Apache-2.0 - apiVersion: kai.scheduler/v1 kind: SchedulingShard metadata: diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/configmap.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/configmap.yaml similarity index 99% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/configmap.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/configmap.yaml index ccb7192ff..86d9bee89 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/configmap.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/configmap.yaml @@ -203,4 +203,3 @@ data: requests: cpu: 50m memory: 64Mi - diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/job.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/job.yaml similarity index 99% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/job.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/job.yaml index 718f02fcb..f6b11ec4b 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/job.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/job.yaml @@ -54,4 +54,3 @@ spec: - name: manifest configMap: name: kai-config-manifest - diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/rbac.yaml similarity index 99% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/rbac.yaml index 0532d2ac3..edad57432 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/rbac.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/kai-config-deployer/rbac.yaml @@ -43,4 +43,3 @@ subjects: - kind: ServiceAccount name: kai-config-deployer namespace: kai-scheduler - diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/post-delete-job.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/post-delete-job.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/post-delete-job.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/post-delete-job.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/crd-upgrader.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/crd-upgrader.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/crd-upgrader.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/crd-upgrader.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/configmap.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/configmap.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/configmap.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/configmap.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/job.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/job.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/job.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/job.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/kai-config.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/kai-config.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/kai-config.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/kai-config.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/build-preemptible.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/build-preemptible.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/build-preemptible.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/build-preemptible.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/build.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/build.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/build.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/build.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/inference.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/inference.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/inference.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/inference.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/train.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/train.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/train.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/train.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/admission-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/admission-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/admission-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/admission-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/admission.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/admission.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/admission.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/admission.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/binder-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/binder-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/binder-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/binder-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/binder.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/binder.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/binder.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/binder.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/crd-manager.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/crd-manager.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/crd-manager.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/crd-manager.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter-binding.yaml similarity index 99% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter-binding.yaml index da8b86ba8..9cfc5580c 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter-binding.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter-binding.yaml @@ -12,7 +12,6 @@ roleRef: kind: ClusterRole name: kai-numa-placement-exporter apiGroup: rbac.authorization.k8s.io - --- # Source: kai-scheduler/templates/rbac/numa-placement-exporter-binding.yaml # Copyright 2026 NVIDIA CORPORATION diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter.yaml similarity index 99% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter.yaml index 158368379..b301bc5ea 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/numa-placement-exporter.yaml @@ -13,7 +13,6 @@ rules: - get - list - patch - --- # Source: kai-scheduler/templates/rbac/numa-placement-exporter.yaml # Copyright 2025 NVIDIA CORPORATION diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/operator-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/operator-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/operator-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/operator-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/operator.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/operator.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/operator.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/operator.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgrouper-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgrouper-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgrouper-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgrouper-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgrouper.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgrouper.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgrouper.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgrouper.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/post-delete-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/post-delete-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/post-delete-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/post-delete-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/post-delete-clusterrole.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/post-delete-clusterrole.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/post-delete-clusterrole.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/post-delete-clusterrole.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scc.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scc.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scc.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scc.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scheduler-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scheduler-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scheduler-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scheduler-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scheduler.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scheduler.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scheduler.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scheduler.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/operator-serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/operator-serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/operator-serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/operator-serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/operator.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/operator.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/operator.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/operator.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/post-delete-serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/post-delete-serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/post-delete-serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/post-delete-serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/resourcereservation-namespace.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/resourcereservation-namespace.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/resourcereservation-namespace.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/resourcereservation-namespace.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/resourcereservation-serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/resourcereservation-serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/resourcereservation-serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/resourcereservation-serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/scalingpod-namespace.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/scalingpod-namespace.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/scalingpod-namespace.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/scalingpod-namespace.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-nvcf-grove-topology/nvcf-cluster-topology/templates/cluster-topology-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-nvcf-grove-topology/nvcf-cluster-topology/templates/cluster-topology-binding.yaml new file mode 100644 index 000000000..ac34bb1be --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-nvcf-grove-topology/nvcf-cluster-topology/templates/cluster-topology-binding.yaml @@ -0,0 +1,36 @@ +--- +# Source: nvcf-cluster-topology/templates/cluster-topology-binding.yaml +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: grove.io/v1alpha1 +kind: ClusterTopologyBinding +metadata: + name: nvcf-mnnvl-topology-binding + labels: + helm.sh/chart: nvcf-cluster-topology-0.1.0 + app.kubernetes.io/name: nvcf-cluster-topology + app.kubernetes.io/instance: nvcf-grove-topology + app.kubernetes.io/version: "0.1.0" + app.kubernetes.io/managed-by: Helm +spec: + levels: + - domain: "gpuclique" + key: "nvidia.com/gpu.clique" + - domain: "hostname" + key: "kubernetes.io/hostname" + schedulerTopologyBindings: + - schedulerName: "kai-scheduler" + topologyReference: "nvcf-mnnvl-topology" diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-nvcf-kai-topology/nvcf-cluster-topology/templates/topology.yaml similarity index 64% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-nvcf-kai-topology/nvcf-cluster-topology/templates/topology.yaml index aa6cfbc70..d285eed1e 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-nvcf-kai-topology/nvcf-cluster-topology/templates/topology.yaml @@ -1,5 +1,5 @@ --- -# Source: helm-nvca-operator/templates/agent-config-merge-cm.yaml +# Source: nvcf-cluster-topology/templates/topology.yaml # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -15,19 +15,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: v1 -kind: ConfigMap +apiVersion: kai.scheduler/v1alpha1 +kind: Topology metadata: - name: agent-config-merge - namespace: nvca-operator + name: nvcf-mnnvl-topology labels: - helm.sh/chart: helm-nvca-operator-1.14.0 - app.kubernetes.io/name: nvca-operator - app.kubernetes.io/instance: nvca-operator - app.kubernetes.io/version: "3.0.4" + helm.sh/chart: nvcf-cluster-topology-0.1.0 + app.kubernetes.io/name: nvcf-cluster-topology + app.kubernetes.io/instance: nvcf-kai-topology + app.kubernetes.io/version: "0.1.0" app.kubernetes.io/managed-by: Helm -data: - config.yaml: |- - cluster: - validationPolicy: - name: Unrestricted +spec: + levels: + - nodeLabel: "nvidia.com/gpu.clique" + - nodeLabel: "kubernetes.io/hostname" diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml new file mode 100644 index 000000000..9bd33f45b --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml @@ -0,0 +1,87 @@ +--- +# Source: helm-nvca-operator/templates/agent-config-merge-cm.yaml +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: agent-config-merge + namespace: nvca-operator + labels: + helm.sh/chart: helm-nvca-operator-1.14.0 + app.kubernetes.io/name: nvca-operator + app.kubernetes.io/instance: nvca-operator + app.kubernetes.io/version: "3.0.4" + app.kubernetes.io/managed-by: Helm +data: + config.yaml: |2 + + cluster: + validationPolicy: + allowedExtraKubernetesTypes: + - group: foo.com + kind: FooType + resource: footypes + version: v1 + - group: scheduling.run.ai + kind: PodGroup + resource: podgroups + version: v2alpha2 + - group: grove.io + kind: PodCliqueSet + resource: podcliquesets + version: v1alpha1 + - group: grove.io + kind: PodClique + resource: podcliques + version: v1alpha1 + - group: grove.io + kind: PodCliqueScalingGroup + resource: podcliquescalinggroups + version: v1alpha1 + - group: scheduler.grove.io + kind: PodGang + resource: podgangs + version: v1alpha1 + - group: nvidia.com + kind: DynamoCheckpoint + resource: dynamocheckpoints + version: v1alpha1 + - group: nvidia.com + kind: DynamoComponentDeployment + resource: dynamocomponentdeployments + version: v1alpha1 + - group: nvidia.com + kind: DynamoGraphDeployment + resource: dynamographdeployments + version: v1alpha1 + - group: nvidia.com + kind: DynamoGraphDeploymentRequest + resource: dynamographdeploymentrequests + version: v1alpha1 + - group: nvidia.com + kind: DynamoGraphDeploymentScalingAdapter + resource: dynamographdeploymentscalingadapters + version: v1alpha1 + - group: nvidia.com + kind: DynamoModel + resource: dynamomodels + version: v1alpha1 + - group: nvidia.com + kind: DynamoWorkerMetadata + resource: dynamoworkermetadatas + version: v1alpha1 + name: Unrestricted diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/chart-defaults-nvcfbackend-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/chart-defaults-nvcfbackend-cm.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/chart-defaults-nvcfbackend-cm.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/chart-defaults-nvcfbackend-cm.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/cluster-validator-network-checks-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/cluster-validator-network-checks-cm.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/cluster-validator-network-checks-cm.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/cluster-validator-network-checks-cm.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/cronjob.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/cronjob.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/cronjob.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/cronjob.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/custom-annotations-configmap.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/custom-annotations-configmap.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/custom-annotations-configmap.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/custom-annotations-configmap.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/custom-network-policies-configmap.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/custom-network-policies-configmap.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/custom-network-policies-configmap.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/custom-network-policies-configmap.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/deployment.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/deployment.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/deployment.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/deployment.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/gpu-profiling-config-configmap.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/gpu-profiling-config-configmap.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/gpu-profiling-config-configmap.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/gpu-profiling-config-configmap.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/helm-managed-nvcfbackend-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/helm-managed-nvcfbackend-cm.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/helm-managed-nvcfbackend-cm.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/helm-managed-nvcfbackend-cm.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/ngc-service-key.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/ngc-service-key.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/ngc-service-key.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/ngc-service-key.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/nvca-operator_rq.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/nvca-operator_rq.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/nvca-operator_rq.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/nvca-operator_rq.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-job.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-job.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-job.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-job.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml new file mode 100644 index 000000000..7193c1c8f --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml @@ -0,0 +1,86 @@ +--- +# Source: helm-nvca-operator/templates/rbac_allowed_extra_types.yaml +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nvca-operator-allowed-extra-types + labels: + helm.sh/chart: helm-nvca-operator-1.14.0 + app.kubernetes.io/name: nvca-operator + app.kubernetes.io/instance: nvca-operator + app.kubernetes.io/version: "3.0.4" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: ["foo.com"] + resources: ["footypes"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["scheduling.run.ai"] + resources: ["podgroups"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["grove.io"] + resources: ["podcliquesets"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["grove.io"] + resources: ["podcliques"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["grove.io"] + resources: ["podcliquescalinggroups"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["scheduler.grove.io"] + resources: ["podgangs"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamocheckpoints"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamocomponentdeployments"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeployments"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeploymentrequests"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeploymentscalingadapters"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamomodels"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamoworkermetadatas"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +--- +# Source: helm-nvca-operator/templates/rbac_allowed_extra_types.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nvca-operator-allowed-extra-types + labels: + helm.sh/chart: helm-nvca-operator-1.14.0 + app.kubernetes.io/name: nvca-operator + app.kubernetes.io/instance: nvca-operator + app.kubernetes.io/version: "3.0.4" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nvca-operator-allowed-extra-types +subjects: +- kind: ServiceAccount + name: nvca-operator + namespace: nvca-operator diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/role.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/role.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/role.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/role.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/role_binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/role_binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/role_binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/role_binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/sa.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/sa.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/sa.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/sa.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml similarity index 97% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml index ca5466e53..b108005c9 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml @@ -42,7 +42,7 @@ data: cloudProvider: "NCP" region: "us-west-1" attributes: [] - capabilities: ["DynamicGPUDiscovery","InfraResourceOverhead","EnforceHelmFunctionResourceLimits","EnforceContainerFunctionResourceLimits","EnforceHelmTaskResourceLimits","EnforceContainerTaskResourceLimits","BYOObservability"] + capabilities: ["DynamicGPUDiscovery","InfraResourceOverhead","EnforceHelmFunctionResourceLimits","EnforceContainerFunctionResourceLimits","EnforceHelmTaskResourceLimits","EnforceContainerTaskResourceLimits","BYOObservability","KAIScheduler","DynamoOperatorSupport"] gpusB64: "" cacheCSIVolumeMountOptionsConfig: disabled: false diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/shutdown-sentinel.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/shutdown-sentinel.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/shutdown-sentinel.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/shutdown-sentinel.yaml diff --git a/docs/user/cluster-management/configuration.md b/docs/user/cluster-management/configuration.md index 2c4b44f22..9c602d973 100644 --- a/docs/user/cluster-management/configuration.md +++ b/docs/user/cluster-management/configuration.md @@ -31,7 +31,7 @@ See below for descriptions of all cluster features. | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | Dynamic GPU Discovery | Enables automatic detection and management of allocatable GPU capacity within the cluster via the NVIDIA GPU Operator. This capability is **strongly recommended** and would only be disabled in cases where [Manual Instance Configuration](./configuration.md) is required. | | Caching Support | Enhances application performance by storing frequently accessed data (models, resources and containers) in a cache. See [cluster-caching](./configuration.md). | -| Optimized AI Workload Scheduling | Enable support for optimized AI workload scheduling using [KAI Scheduler](https://github.com/kai-scheduler/KAI-Scheduler). Additional setup details: [KAI Scheduler](./kai-scheduler.md) | +| Optimized AI Workload Scheduling | Enables KAI Scheduler for GPU bin-packing and queues. See [KAI Scheduler](./kai-scheduler.md), [Gang Scheduling](./gang-scheduling.md), and [Topology-Aware Scheduling](./topology-aware-scheduling.md). | | Shared Cluster mode | Partitions the Kubernetes cluster's nodes into NVCF and non-NVCF pools. Set the label `nvca.nvcf.nvidia.io/schedule=true` on all nodes that can receive NVCF workload Pods. **Note**: this is an advanced use case and should not be used unless absolutely necessary | @@ -234,6 +234,11 @@ Additional prerequisites: \- The [NVIDIA GPU DRA driver](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-intro-install.html) must be installed. \- The `NVLinkOptimized` cluster attribute must be added during cluster registration. +See [Topology-Aware Scheduling](./topology-aware-scheduling.md) to place +multi-node workloads in one GPU clique. Use +[Gang Scheduling](./gang-scheduling.md) when every Pod must be placed +atomically. + In NVLink-optimized mode, the NVIDIA GPU DRA driver currently limits one GPU-enabled Pod to a node. To optimally utilize these clusters, GPU-enabled Pods _should_ request a full node's worth of GPUs. For example, nodes in GB200 clusters have 4 GPUs each so all containers and all GPU-enabled Pods in a workload must request `nvidia.com/gpu` values that sum to 4. diff --git a/docs/user/cluster-management/gang-scheduling.md b/docs/user/cluster-management/gang-scheduling.md new file mode 100644 index 000000000..623a38730 --- /dev/null +++ b/docs/user/cluster-management/gang-scheduling.md @@ -0,0 +1,160 @@ +# Gang Scheduling + +Gang scheduling holds a group of Pods until the scheduler can place every +required member. This prevents a partial deployment from consuming GPUs while +the remaining Pods stay `Pending`. + +NVCF Helm functions and tasks may use custom resources for several components +for gang-scheduled workloads: + +- [KAI Scheduler](https://github.com/kai-scheduler/KAI-Scheduler) provides + queueing, resource allocation, and atomic placement through `PodGroup` + resources. +- [Grove](https://github.com/NVIDIA/grove) represents related workload roles as + `PodCliqueSet`, `PodClique`, `PodCliqueScalingGroup`, and `PodGang` + resources. Grove delegates placement to KAI Scheduler. +- [Dynamo](https://docs.nvidia.com/dynamo/v1.2.1/kubernetes-deployment/scale/grove) + describes inference services such as frontend, prefill, and decode workers + in a `DynamoGraphDeployment`. The Dynamo operator uses Grove to orchestrate + those services. + +The NVCF Cluster Agent (NVCA) admits the KAI, Grove, and Dynamo resources in +function Helm charts when their compute plane add-ons are enabled. +NVCA also uses KAI implicitly when the `KAIScheduler` feature flag is enabled +to binpack workloads. + +For more background, see the upstream +[KAI gang scheduling guide](https://github.com/kai-scheduler/KAI-Scheduler/blob/main/docs/batch/README.md) +and [Grove core concepts](https://github.com/NVIDIA/grove/blob/main/docs/user-guide/01_core-concepts/01_overview.md). + +## Enable gang scheduling + +Enable KAI Scheduler, Grove or Dynamo when functions use their custom resources: + +```yaml +addons: + kaiScheduler: + enabled: true + groveOperator: + enabled: true + dynamoOperator: + enabled: true +``` + +The compute plane installs the components in dependency order: + +1. KAI Scheduler +2. Grove +3. Dynamo + +The same add-ons configure NVCA: + +- KAI adds the `KAIScheduler` feature gate and permits `PodGroup`. +- Grove permits `PodCliqueSet`, `PodClique`, `PodCliqueScalingGroup`, and + `PodGang`. Grove requires KAI Scheduler to be enabled. +- Dynamo adds the `DynamoOperatorSupport` feature gate and permits Dynamo + custom resources. Dynamo requires Grove to be enabled. + +See [KAI Scheduler](./kai-scheduler.md) for queue configuration and the +standalone installation path. + +## Gang schedule a StatefulSet with KAI + +Use a StatefulSet with parallel Pod management for a direct KAI workload: + +```yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: distributed-worker +spec: + podManagementPolicy: Parallel + replicas: 2 + selector: + matchLabels: + app: distributed-worker + template: + metadata: + labels: + app: distributed-worker + spec: + containers: + - name: worker + image: +``` + +When the `KAIScheduler` feature is enabled, NVCA assigns the KAI scheduler and +queue to the Pod template. KAI creates one `PodGroup` for the StatefulSet and +waits until every replica can be placed. + +Add topology annotations only when the gang must also fit in a specific +hardware domain. See [Topology-Aware Scheduling](./topology-aware-scheduling.md). + + +KAI creates a separate `PodGroup` for each Deployment replica. Use a +StatefulSet when all replicas must be scheduled as one gang. + + +The +[multi-node Helm function sample](https://github.com/NVIDIA/nvcf/tree/main/examples/function-samples/helmchart-samples/multi-node-helm-function-test) +contains optional KAI topology annotations for gangs that also need GPU clique +placement. + +## Gang schedule a Dynamo workload with Grove + +A `DynamoGraphDeployment` groups inference roles in one resource. A multinode +service tells Dynamo and Grove that each replica needs multiple Pods: + +```yaml +apiVersion: nvidia.com/v1alpha1 +kind: DynamoGraphDeployment +metadata: + name: myllm +spec: + services: + VllmDecodeWorker: + componentType: worker + replicas: 1 + multinode: + nodeCount: 2 + resources: + limits: + gpu: "4" +``` + +The Dynamo operator converts the service into Grove resources. Grove groups the +Pods, and KAI reserves and places the complete group. Function authors do not explicitly +create `PodGang` or `PodGroup` resources for a `DynamoGraphDeployment`. + +See the [Dynamo Operator sample](https://github.com/NVIDIA/nvcf/tree/main/examples/function-samples/helmchart-samples/dynamo-operator-sample) +for a disaggregated frontend, prefill, and decode workload. See the upstream +[Dynamo multinode guide](https://docs.nvidia.com/dynamo/v1.2.1/kubernetes-deployment/scale/multinode-deployments) +and [Grove quickstart](https://github.com/NVIDIA/grove/blob/main/docs/quickstart.md) +for advanced workload configuration. + +## Verify gang scheduling (Cluster admins only) + +Check that the operators are running: + +```bash +kubectl get pods -n kai-scheduler +kubectl get pods -n grove-system +kubectl get pods -n dynamo-system +``` + +Inspect the scheduling resources for a deployed function: + +```bash +kubectl get podgroups.scheduling.run.ai -A +kubectl get podgangs.scheduler.grove.io -A +kubectl get podcliquesets.grove.io -A +``` + +If a gang remains `Pending`, describe its Pods and KAI `PodGroup`. Common +causes are insufficient free GPUs, queue limits, or topology constraints that +no available domain can satisfy. + +```bash +kubectl describe pod -n +kubectl describe podgroup -n +``` diff --git a/docs/user/cluster-management/kai-scheduler.md b/docs/user/cluster-management/kai-scheduler.md index 78af4db19..060bde3c4 100644 --- a/docs/user/cluster-management/kai-scheduler.md +++ b/docs/user/cluster-management/kai-scheduler.md @@ -1,32 +1,35 @@ # KAI Scheduler Integration Guide -[KAI Scheduler](https://github.com/kai-scheduler/KAI-Scheduler) is an open source Kubernetes Native scheduler for AI workloads at large scale. -To use the KAI Scheduler for NVCF Workloads the following configuration should be applied post the installation of the KAI Scheduler in the cluster and the [Optimized AI Workload Scheduling](./configuration.md) enabled on the -cluster. NVCF Workloads deployed will be automatically BinPacked upon this cluster configuration changes. +[KAI Scheduler](https://github.com/kai-scheduler/KAI-Scheduler) is an open +source Kubernetes scheduler for AI workloads. NVCF uses it for GPU bin-packing, +queues, gang scheduling, and topology-aware placement. -**KAI Scheduler Installation** +## Install KAI Scheduler - Upgrade to latest [KAI Scheduler release](https://github.com/kai-scheduler/KAI-Scheduler/releases) is recommended to get latest fixes and security patches - +Use a tested [KAI Scheduler release](https://github.com/kai-scheduler/KAI-Scheduler/releases) +that is compatible with the NVCF compute plane stack. -When you enable `addons.kaiScheduler.enabled` in the `nvcf-compute-plane` Helmfile stack, the stack installs KAI Scheduler for you (release and namespace `kai-scheduler`). Enable that flag whenever you enable Grove or Dynamo. Skip the manual install below in that case. +Set `addons.kaiScheduler.enabled` in the `nvcf-compute-plane` Helmfile +environment to install KAI Scheduler as release and namespace `kai-scheduler`. +Grove, Dynamo, and topology-aware scheduling require this add-on. Skip the +manual installation below when the add-on is enabled. -Use the manual install when you need KAI without the compute-plane add-on, for example when enabling only the NVCA `KAIScheduler` feature gate. +Use the manual path when KAI is managed outside the compute plane stack. -NVCA's KAI scheduler integration expects default queues to exist with names `default-parent-queue` (parent) and `default-queue` (child); -other queues may exist in the cluster. +NVCA expects a parent queue named `default-parent-queue` and a child queue +named `default-queue`. Other queues may also exist. -One caveat is that NVCA expects all queues used to create NVCF workloads to have unlimited (`-1`) quotas and limits -to ensure full cluster capacity utilization and accurate usage tracking. If the cluster is partitioned to serve both NVCF and non-NVCF workloads -and KAI scheduler queue quotas/limits are limited to reflect this, then [Shared Cluster mode](./configuration.md#cluster-features) must be enabled so non-NVCF workload nodes -are accurately excluded from tracking and scheduling by NVCA. - +Set unlimited (`-1`) quotas and limits on every queue used for NVCF workloads. +This lets NVCA track the complete cluster capacity. If NVCF and non-NVCF +workloads share a cluster with limited KAI queues, enable +[Shared Cluster mode](./configuration.md#cluster-features) so NVCA excludes +non-NVCF nodes from capacity tracking and scheduling. -Create `values.yaml` with [default queue](https://raw.githubusercontent.com/NVIDIA/KAI-Scheduler/refs/heads/main/docs/quickstart/default-queues.yaml) attributes: +Create `values.yaml` with the required default queues: ```yaml title="kai-scheduler-queues.yaml" @@ -79,3 +82,15 @@ defaultQueue: ```bash helm install kai-scheduler oci://ghcr.io/kai-scheduler/kai-scheduler/kai-scheduler -f values.yaml -n kai-scheduler --create-namespace --version v0.14.0 ``` + +## Schedule multi-Pod workloads + +KAI can hold a multi-Pod workload until all required members fit. Grove and +Dynamo build on this behavior for multi-role inference services. See +[Gang Scheduling](./gang-scheduling.md) for add-on configuration, workload +examples, supported resource types, and troubleshooting. + +On NVLink-optimized clusters, KAI can also place the complete gang in one GPU +clique. See +[Topology-Aware Scheduling](./topology-aware-scheduling.md) for GPU DRA +prerequisites, topology configuration, Grove bindings, and function examples. diff --git a/docs/user/cluster-management/self-managed.md b/docs/user/cluster-management/self-managed.md index 7d3ae5a57..7827ee4eb 100644 --- a/docs/user/cluster-management/self-managed.md +++ b/docs/user/cluster-management/self-managed.md @@ -31,7 +31,11 @@ Before installing the NVCA Operator, ensure the following prerequisites are met: - The [NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) is installed on the GPU cluster. The GPU Operator manages the NVIDIA drivers, device plugin, and GPU feature discovery required for workload scheduling. For development or testing environments without physical GPUs, see [fake-gpu-operator](../fake-gpu-operator). -- (Optional) The [KAI Scheduler](./kai-scheduler.md) can be installed on the GPU cluster for optimized AI workload scheduling and bin-packing of GPU resources. It is required only when you enable the `KAIScheduler` feature flag. See [NVCA Configuration](./configuration.md). +- (Optional) Install [KAI Scheduler](./kai-scheduler.md) for GPU bin-packing + and queues. KAI is also the scheduling foundation for + [gang scheduling](./gang-scheduling.md) and + [topology-aware scheduling](./topology-aware-scheduling.md) with Grove and + Dynamo. - `GPU Workload Components` must be available in a user-managed registry that your Kubernetes cluster can access. See `GPU Workload Components` under [self-hosted-artifact-manifest](../manifest.md) for necessary artifacts and [self-hosted-image-mirroring](../image-mirroring.md) for mirroring instructions. diff --git a/docs/user/cluster-management/topology-aware-scheduling.md b/docs/user/cluster-management/topology-aware-scheduling.md new file mode 100644 index 000000000..d0dbeb256 --- /dev/null +++ b/docs/user/cluster-management/topology-aware-scheduling.md @@ -0,0 +1,241 @@ +# Topology-Aware Scheduling + +Topology-aware scheduling places related Pods in a hardware domain with the +network bandwidth they require. On an NVLink-optimized cluster, NVCF can place +a multi-node function inside one GPU clique instead of spreading its Pods +across slower links. + +The scheduling path has four layers: + +1. The NVIDIA GPU DRA driver labels nodes with `nvidia.com/gpu.clique`, + identifying a particular NVLink partition. +2. KAI Scheduler reads a cluster-scoped `Topology` and places a gang in the + requested domain. +3. Grove maps its workload hierarchy to the KAI topology through a `ClusterTopologyBinding`. +4. Dynamo uses Grove for placement of frontend, prefill, decode, and other + inference services. + +See the +[NVIDIA GPU DRA ComputeDomain guide](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-cds.html), +[KAI topology guide](https://github.com/kai-scheduler/KAI-Scheduler/blob/main/docs/topology/README.md), +[Grove topology guide](https://github.com/NVIDIA/grove/blob/main/docs/user-guide/topology-aware-scheduling.md), +and [Dynamo topology guide](https://docs.nvidia.com/dynamo/v1.2.1/kubernetes-deployment/scale/topology-aware-scheduling) +for component-level details. + +## Prerequisites + +- Install the + [NVIDIA GPU DRA driver](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-intro-install.html). +- Request a full node of GPUs for each GPU-enabled Pod in your Helm chart. + The GPU DRA driver currently supports one GPU-enabled Pod per node in this mode. + +See [NVLink-optimized clusters](./configuration.md#nvlink-optimized-clusters) +for cluster registration and GPU request requirements. + +### Use KAI Scheduler only + +Enable KAI Scheduler in the compute plane stack during installation: + +```yaml +addons: + topologyAwareScheduling: + enabled: true + kaiScheduler: + enabled: true + groveOperator: + enabled: false + dynamoOperator: + enabled: false +``` + +Topology levels are ordered from the widest domain to the narrowest domain. +The compute plane automatically creates this KAI resource on installation: + +```yaml +apiVersion: kai.scheduler/v1alpha1 +kind: Topology +metadata: + name: nvcf-mnnvl-topology +spec: + levels: + - nodeLabel: nvidia.com/gpu.clique + - nodeLabel: kubernetes.io/hostname +``` + +Enabling `addons.topologyAwareScheduling` without +`addons.kaiScheduler.enabled` stops Helmfile rendering with an error. + +#### Example: place a StatefulSet in one GPU clique + +Add the KAI topology annotations to the StatefulSet object: + +```yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + kai.scheduler/topology: "nvcf-mnnvl-topology" + kai.scheduler/topology-required-placement: "nvidia.com/gpu.clique" +spec: + podManagementPolicy: Parallel + replicas: 2 +``` + +KAI waits until all replicas fit in one value of +`nvidia.com/gpu.clique`. This is also a gang-scheduling request. See +[Gang Scheduling](./gang-scheduling.md) for atomic placement behavior and +Grove or Dynamo workloads. + +Creating a `Topology` does not change workloads that do not opt in. + +## Use Grove and/or Dynamo + +Enable the complete scheduling stack during compute stack installation +when functions use Grove or Dynamo: + +```yaml +addons: + topologyAwareScheduling: + enabled: true + kaiScheduler: + enabled: true + groveOperator: + enabled: true + dynamoOperator: + enabled: true +``` + +The `topologyAwareScheduling` toggle enables topology handling in Grove and +automatically creates a binding for each configured KAI topology on installation: + +```yaml +apiVersion: grove.io/v1alpha1 +kind: ClusterTopologyBinding +metadata: + name: nvcf-mnnvl-topology-binding +spec: + levels: + - domain: gpuclique + key: nvidia.com/gpu.clique + - domain: hostname + key: kubernetes.io/hostname + schedulerTopologyBindings: + - schedulerName: kai-scheduler + topologyReference: nvcf-mnnvl-topology +``` + +See the +[Grove automatic MNNVL guide](https://github.com/NVIDIA/grove/blob/main/docs/user-guide/auto-mnnvl.md) +for more details. + +### Example: place a Grove workload in one GPU clique + +Grove workload authors use the portable `gpuclique` domain from the generated +binding instead of the cluster's node label: + +```yaml +apiVersion: grove.io/v1alpha1 +kind: PodCliqueSet +metadata: + name: distributed-inference +spec: + replicas: 1 + template: + topologyConstraint: + topologyName: nvcf-mnnvl-topology-binding + pack: + required: gpuclique + cliques: + - name: worker + spec: + roleName: worker + replicas: 2 + minAvailable: 2 + podSpec: + containers: + - name: worker + image: + resources: + limits: + nvidia.com/gpu: "4" +``` + +Grove translates `gpuclique` to `nvidia.com/gpu.clique` and sends the +constraint to KAI. Child `PodClique` or `PodCliqueScalingGroup` constraints +can select the same or a narrower domain. See the +[Grove topology constraint rules](https://github.com/NVIDIA/grove/blob/main/docs/user-guide/topology-aware-scheduling.md#topology-constraint-rules). + +### Example: place an entire DynamoGraphDeployment in one GPU clique + +This `DynamoGraphDeployment` will create Grove pod cliques for GPU and frontend workers +that get scheduled in a single `gpuclique` domain: + +```yaml +apiVersion: nvidia.com/v1alpha1 +kind: DynamoGraphDeployment +metadata: + name: my-llm +spec: + topologyConstraint: + topologyProfile: nvcf-mnnvl-topology-binding + packDomain: gpuclique + services: + VllmWorker: + componentType: worker + replicas: 2 + envFromSecret: hf-token-secret + resources: + limits: + gpu: "1" + extraPodSpec: + mainContainer: + image: my-image + command: ["/bin/sh", "-c"] + args: + - python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B + Frontend: + componentType: frontend + replicas: 1 + extraPodSpec: + mainContainer: + image: my-image + command: ["/bin/sh", "-c"] + args: + - python3 -m dynamo.frontend +``` + +In general Dynamo creates Grove resources from a `DynamoGraphDeployment`, which Grove +and KAI place. Before adding +topology constraints to a `DynamoGraphDeployment`, see the +[topology-aware scheduling guide for Dynamo 1.2.1](https://docs.nvidia.com/dynamo/v1.2.1/kubernetes-deployment/scale/topology-aware-scheduling) +to check the workload fields and topology resources expected by the compute +plane stack's pinned operator. + +## Verify topology resources (Cluster admins only) + +Confirm that nodes have GPU clique labels: + +```bash +kubectl get nodes -L nvidia.com/gpu.clique +``` + +Confirm that the KAI topology exists: + +```bash +kubectl get topologies.kai.scheduler nvcf-mnnvl-topology +``` + +When Grove is enabled, confirm that its binding exists: + +```bash +kubectl get clustertopologybindings.grove.io \ + nvcf-mnnvl-topology-binding +``` + +If Pods remain `Pending`, verify that one clique has enough free nodes for the +entire gang. Then inspect the Pod and scheduler events: + +```bash +kubectl describe pod -n +kubectl describe podgroup -n +``` diff --git a/docs/user/helm-functions.md b/docs/user/helm-functions.md index ecd2b748a..b2f08be06 100644 --- a/docs/user/helm-functions.md +++ b/docs/user/helm-functions.md @@ -9,7 +9,7 @@ Ensure that your helm charts version does not contain `-` For example `v1` is ok -1. The helm chart **must have a "mini-service" container defined, which will be used as the inference entry point.** +1. The Helm chart must define a `mini-service` container as the inference entry point. 2. The name of this service in your helm chart should be supplied by setting `helmChartServiceName` during the function definition. This allows Cloud Functions to communicate and make inference requests to the "mini-service" endpoint. @@ -35,11 +35,13 @@ All Pod specs in your helm chart will be updated with pull secrets at runtime, s - The `helmChart` property should be set to the OCI URL of the helm chart that will deploy the "mini-service". The helm chart URL should follow the format: `oci://${REGISTRY}/${REPOSITORY}/charts/$NAME-X.Y.Z.tgz`. The chart name should not contain `-` in the version string. - - The `helmChartServiceName` is used for checking if the "mini-service" is ready for inference and is also scraped for function metrics. At this time, templatized service names are not supported. **This must match the service name of your "mini-service" with the exposed entry point port.** + - NVCF uses `helmChartServiceName` for readiness checks and function + metrics. It must match the service that exposes the `mini-service` entry + point. Templated service names are not supported. - Important: The Helm chart name should not contain underscores or other special symbols, as that may cause issues during deployment. -**Example Creation via API** +### API example Please see our [sample helm chart used](https://github.com/NVIDIA/nvcf/tree/main/examples/function-samples/helmchart-samples/inference-test-sample) in this example for reference. @@ -68,12 +70,34 @@ For gRPC-based functions, set `"inferenceURL" : "/gRPC"`. This signals to Cloud 3. Proceed with function deployment and invocation normally. -**Multi-node helm deployment** +### Multi-node helm deployment + To create a multi-node helm deployment, you need to use the following format for the `instanceType`: `.GPU._x[.x]`. For example, `DGXC.GPU.L40S_1x` is a single L40S instance while `ON-PREM.GPU.B200_8x.x2` is two full nodes of 8-way B200. A sample helm chart for a multi-node deployment can be found [in the multi-node helm example](https://github.com/NVIDIA/nvcf/tree/main/examples/function-samples/helmchart-samples/multi-node-helm-function-test/). +#### Multi-node NVLink scheduling + + +The compute cluster must have the `NVLinkOptimized` attribute. See +[NVLink-optimized clusters](./cluster-management/configuration.md#nvlink-optimized-clusters). + + +NVCF can place the Pods from one multi-node Helm function in a single NVLink +GPU clique. Use [Gang Scheduling](./cluster-management/gang-scheduling.md) when every Pod +must be placed atomically. Use +[Topology-Aware Scheduling](./cluster-management/topology-aware-scheduling.md) +to place that gang in one GPU clique. The guides include examples for direct +KAI StatefulSets and workloads managed through Grove and Dynamo. +NVCA will create a +[`ComputeDomain`](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-cds.html) +to connect GPU workload Pods through IMEX. + + +Each GPU-enabled Pod must request a full node of GPUs. + + ## Limitations When using Helm Charts to deploy a function, the following limitations need to be taken into consideration. @@ -98,7 +122,7 @@ it will only do so if modification will not break your chart when it is installe Possible areas amenable to modification will be noted in the restrictions section below. Any standard that cannot be enforced by modification will result in error(s) during function creation. -**Restrictions** +#### Restrictions - Supported k8s artifacts under Helm Chart Namespace are listed below; others will be rejected: - ConfigMaps diff --git a/examples/function-samples/helmchart-samples/dynamo-operator-sample/README.md b/examples/function-samples/helmchart-samples/dynamo-operator-sample/README.md index 75c27aa24..4dc92fd10 100644 --- a/examples/function-samples/helmchart-samples/dynamo-operator-sample/README.md +++ b/examples/function-samples/helmchart-samples/dynamo-operator-sample/README.md @@ -1,17 +1,29 @@ # Dynamo Operator on NVCA self-hosted demo -This demo uses and modifies [this upstream example disaggregated router DGD](https://github.com/ai-dynamo/dynamo/blob/v1.0.2/examples/backends/vllm/deploy/disagg_router.yaml). +This demo adapts an +[upstream disaggregated router example](https://github.com/ai-dynamo/dynamo/blob/v1.0.2/examples/backends/vllm/deploy/disagg_router.yaml) +for an NVCF Helm function. Dynamo uses Grove and KAI Scheduler to orchestrate +and place the workload. See +[Gang Scheduling](../../../../docs/user/cluster-management/gang-scheduling.md) +and +[Topology-Aware Scheduling](../../../../docs/user/cluster-management/topology-aware-scheduling.md) +for the production compute plane configuration. ## Prerequisites -* An NVCF self-hosted Kubernetes cluster with Dynamo Operator installed (see [the local Dynamo Operator install guide](../../../../tools/ncp-local-cluster/docs/dynamo-operator.md) for local cluster testing). -* A HuggingFace [token](https://huggingface.co/docs/hub/en/security-tokens), if using the default model [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) used in the example Helm chart +- An NVCF self-hosted Kubernetes cluster with the Dynamo, Grove, and KAI + Scheduler add-ons installed. For local cluster testing, see the + [local Dynamo Operator guide](../../../../tools/ncp-local-cluster/docs/dynamo-operator.md). +- A Hugging Face [token](https://huggingface.co/docs/hub/en/security-tokens) + when using the example Helm chart's default + [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) model. ## Deploying your Dynamo chart 1. Package the chart and push it to an OCI registry your cluster can reach, then register pull credentials with `nvcf-cli`: - **Note:** ensure your DGD name is <= 24 characters long, otherwise the generated object names will be too long for Kubernetes to handle + Note: Keep the DGD name at 24 characters or fewer so generated Kubernetes + object names remain valid. ```console $ helm package dynamo-operator-test @@ -26,7 +38,8 @@ This demo uses and modifies [this upstream example disaggregated router DGD](htt 1. Create the function, using the below payload as an example, substituting in your Helm chart's values - **Note:** the `helmChartServiceName` field must contain a string of the format `-frontend`. In this example, the DGD is named `myllm`, so the service name generated is `myllm-frontend`. + Note: Set `helmChartServiceName` to `-frontend`. This example + names the DGD `myllm`, so the generated service is `myllm-frontend`. ```console $ cat < function-create.json diff --git a/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml b/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml index ac23a0695..648354efa 100644 --- a/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml +++ b/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml @@ -22,7 +22,13 @@ apiVersion: apps/v1 kind: StatefulSet metadata: annotations: - # dra.nvcf.nvidia.io/required-nvlink-domain-index: "0" # force all nodes to use the same NVLink domain + # Gang scheduling on an NVLink-optimized cluster running KAI Scheduler. KAI + # reads these from the StatefulSet, not the pod template, and holds every + # replica until they all fit in one clique. Requires the cluster operator to + # have installed the nvcf-mnnvl-topology Topology resource. + # + # kai.scheduler/topology: "nvcf-mnnvl-topology" + # kai.scheduler/topology-required-placement: "nvidia.com/gpu.clique" name: {{ include "multi-node-test.name" . }} labels: {{- include "multi-node-test.labels" . | nindent 4 }} diff --git a/fern/versions/dev.yml b/fern/versions/dev.yml index d2dce816a..f3ec7ca54 100644 --- a/fern/versions/dev.yml +++ b/fern/versions/dev.yml @@ -67,6 +67,10 @@ navigation: path: ../../docs/user/cluster-management/multi-tenancy.md - page: KAI Scheduler path: ../../docs/user/cluster-management/kai-scheduler.md + - page: Gang Scheduling + path: ../../docs/user/cluster-management/gang-scheduling.md + - page: Topology-Aware Scheduling + path: ../../docs/user/cluster-management/topology-aware-scheduling.md - section: Low Latency Streaming skip-slug: true contents: