diff --git a/api/worker/v1alpha1/generated.proto b/api/worker/v1alpha1/generated.proto index c33ac0fe..38d1a190 100644 --- a/api/worker/v1alpha1/generated.proto +++ b/api/worker/v1alpha1/generated.proto @@ -294,7 +294,7 @@ message InstanceResources { optional int32 acceleratorSlicedMemoryPercentage = 5; // AcceleratorSlicedCoresPercentage is the per-card compute (SM) budget requested on - // a sliced InstanceType, as a percentage in [0,100]. It must not be smaller than + // a sliced InstanceType, as a percentage in [0,100]. It is independent of // AcceleratorSlicedMemoryPercentage; when only one of the two is set the webhook // copies it to the other. It is ignored by non-sliced requests. // @@ -648,7 +648,7 @@ message InstanceTypeStatus { // InstanceTypeUnitResources describes the unit resources of the InstanceType. message InstanceTypeUnitResources { - // CPU is the unit CPU resource(milli-cores) of the InstanceType. + // CPU is the unit CPU resource(cores) of the InstanceType. optional string cpu = 1; // RAM is the unit RAM resource(Mi) of the InstanceType. diff --git a/api/worker/v1alpha1/instance.go b/api/worker/v1alpha1/instance.go index 03152eda..dfdd5490 100644 --- a/api/worker/v1alpha1/instance.go +++ b/api/worker/v1alpha1/instance.go @@ -173,7 +173,7 @@ type InstanceResources struct { AcceleratorSlicedMemoryPercentage int32 `json:"acceleratorSlicedMemoryPercentage,omitempty" protobuf:"varint,5,opt,name=acceleratorSlicedMemoryPercentage"` // nolint: lll // AcceleratorSlicedCoresPercentage is the per-card compute (SM) budget requested on - // a sliced InstanceType, as a percentage in [0,100]. It must not be smaller than + // a sliced InstanceType, as a percentage in [0,100]. It is independent of // AcceleratorSlicedMemoryPercentage; when only one of the two is set the webhook // copies it to the other. It is ignored by non-sliced requests. // diff --git a/api/worker/v1alpha1/zz_generated.crds.go b/api/worker/v1alpha1/zz_generated.crds.go index 1b3347d0..ca8b3766 100644 --- a/api/worker/v1alpha1/zz_generated.crds.go +++ b/api/worker/v1alpha1/zz_generated.crds.go @@ -568,7 +568,7 @@ func crd_gpustack_api_worker_v1alpha1_Instance() *v1.CustomResourceDefinition { XIntOrString: true, }, "acceleratorSlicedCoresPercentage": { - Description: "AcceleratorSlicedCoresPercentage is the per-card compute (SM) budget requested on\na sliced InstanceType, as a percentage in [0,100]. It must not be smaller than\nAcceleratorSlicedMemoryPercentage; when only one of the two is set the webhook\ncopies it to the other. It is ignored by non-sliced requests.", + Description: "AcceleratorSlicedCoresPercentage is the per-card compute (SM) budget requested on\na sliced InstanceType, as a percentage in [0,100]. It is independent of\nAcceleratorSlicedMemoryPercentage; when only one of the two is set the webhook\ncopies it to the other. It is ignored by non-sliced requests.", Type: "integer", Format: "int32", }, @@ -1123,7 +1123,7 @@ func crd_gpustack_api_worker_v1alpha1_InstanceType() *v1.CustomResourceDefinitio Type: "object", Properties: map[string]v1.JSONSchemaProps{ "cpu": { - Description: "CPU is the unit CPU resource(milli-cores) of the InstanceType.", + Description: "CPU is the unit CPU resource(cores) of the InstanceType.", Type: "string", }, "ram": { diff --git a/api/worker/zz_generated.openapi.go b/api/worker/zz_generated.openapi.go index a453e210..4e597507 100644 --- a/api/worker/zz_generated.openapi.go +++ b/api/worker/zz_generated.openapi.go @@ -2643,7 +2643,7 @@ func schema_gpustack_api_worker_v1alpha1_InstanceResources(ref common.ReferenceC }, "acceleratorSlicedCoresPercentage": { SchemaProps: spec.SchemaProps{ - Description: "AcceleratorSlicedCoresPercentage is the per-card compute (SM) budget requested on a sliced InstanceType, as a percentage in [0,100]. It must not be smaller than AcceleratorSlicedMemoryPercentage; when only one of the two is set the webhook copies it to the other. It is ignored by non-sliced requests.", + Description: "AcceleratorSlicedCoresPercentage is the per-card compute (SM) budget requested on a sliced InstanceType, as a percentage in [0,100]. It is independent of AcceleratorSlicedMemoryPercentage; when only one of the two is set the webhook copies it to the other. It is ignored by non-sliced requests.", Type: []string{"integer"}, Format: "int32", }, @@ -3769,7 +3769,7 @@ func schema_gpustack_api_worker_v1alpha1_InstanceTypeUnitResources(ref common.Re Properties: map[string]spec.Schema{ "cpu": { SchemaProps: spec.SchemaProps{ - Description: "CPU is the unit CPU resource(milli-cores) of the InstanceType.", + Description: "CPU is the unit CPU resource(cores) of the InstanceType.", Type: []string{"string"}, Format: "", }, diff --git a/docs/architecture.md b/docs/architecture.md index 015734a9..52f3aa08 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -245,7 +245,7 @@ flowchart LR **Three-view status.** `InstanceType.status` carries three per-card bin-packing projections computed from the `Devices` ledger (not a credits fold-down): `Accelerator` = free whole cards; `AcceleratorShared` = shareable ownership slots; `AcceleratorSliced` = sliceable VRAM-percent units. Because the reconciler watches the `Devices` CR and writes the result into a real CRD's `.status`, a `kubectl get instancetype -w` observes capacity move as pods allocate and free — a read-only projection over the ClusterQueue could not (it borrows the CQ `resourceVersion`, unchanged on a `Devices`-only allocation). The v1 (`worker.gpustack.ai/v1`) InstanceType is a thin proxy + conversion over the real `v1alpha1` CRD; the unit spec lives **only** on the InstanceType — a derived type is stamped with the fixed default at creation, and an admin edit changes only the InstanceType (never a Node or the ClusterQueue notes). The InstanceType validating webhook **requires** the complete input on create — `acceleratorGroup` (only when `acceleratable`) / `os` / `arch` plus the unit triple (`unitResources.cpu`/`.ram` + `localStorage`; an empty or partial spec is rejected), read independently of any editable setting, and a **CPU-only (`acceleratable=false`) type's unit CPU must be exactly 1 core** (an accelerated type keeps any unitless positive integer) — and **freezes the spec on update**: every field is **immutable except `displayName` (rename) and `inactive` (take in/out of service)**, so to re-size or re-point a pool you delete and re-create the type. Create-time shape checks are not re-run on update (only immutability is), so a legacy type stored before a tightened rule can still be renamed or deactivated. The defaulting webhook defaults an empty `generalGroup` to the `generic` sentinel, stamps the pool's schedule labels (`nodefeature.PoolScheduleLabels`, grouped by `instance-type-aware-cpu-manufacturer`) and the `schedule.gpustack.ai/queue-entrance` label (`nodefeature.FormatLocalQueueName(name)`), enriches the descriptors from a matching ResourceFlavor, and — when awareness is on — folds that flavor's `cpuDetail` note into `spec.cpu` (generic) or `spec.accelerator.cpu` (accelerated). The Instance validating webhook enforces the unit spec on **Create and Update**: a submission's RAM must not exceed `unitRAM × count` and its local storage must not exceed the InstanceType's `LocalStorage`. The Pod webhook reads the **InstanceType** for a slice's per-card VRAM divisor — reverse-looked-up by the `schedule.gpustack.ai/queue-entrance` label to `spec.memory` (Stage 4, Pod admission webhook) — never the user-writable LocalQueue. -**Pod admission webhook.** A `pods` CREATE webhook (objectSelector on `kueue.x-k8s.io/queue-name`, `failurePolicy: Fail`) mutates a sliced request — defaulting `.sliced.cores-percentage` to 100 and folding `.sliced.memory-percentage` / `.sliced.memory-mib` into `.sliced.units` (only when units are absent; memory-percentage wins) — and validates it (rejecting no-memory / both-memory-keys / non-positive `.sliced.*` / `cores-% < memory-%` / mixed modes). The per-card VRAM divisor for the fold is read from the operator-owned **InstanceType**'s `spec.memory` (reverse-looked-up by the `schedule.gpustack.ai/queue-entrance` label), never from the user-writable LocalQueue. Its `MutatingWebhookConfiguration` name sorts before `kueue-mutating-webhook-configuration` so the fold runs before Kueue hashes container resources. +**Pod admission webhook.** A `pods` CREATE webhook (objectSelector on `kueue.x-k8s.io/queue-name`, `failurePolicy: Fail`) mutates a sliced request — defaulting `.sliced.cores-percentage` to 100 and folding `.sliced.memory-percentage` / `.sliced.memory-mib` into `.sliced.units` (only when units are absent; memory-percentage wins) — and validates it (rejecting no-memory / both-memory-keys / non-positive `.sliced.*` / mixed modes). The per-card VRAM divisor for the fold is read from the operator-owned **InstanceType**'s `spec.memory` (reverse-looked-up by the `schedule.gpustack.ai/queue-entrance` label), never from the user-writable LocalQueue. Its `MutatingWebhookConfiguration` name sorts before `kueue-mutating-webhook-configuration` so the fold runs before Kueue hashes container resources. **Running-instance stop.** Before (re)creating an Instance's Pod, the `InstanceReconciler` (`instance.go`) reads the backing `ClusterQueue`'s `StopPolicy` and **stops** the Instance (`spec.stop=true`) — rather than recreating a Pod the queue can never admit — when that queue is `HoldAndDrain` (a pool drain or a teardown that evicts admitted workloads), or when the `InstanceType` is being deleted or gone. An admin `Hold` (the `Inactive` switch) is deliberately **not** a stop: already-running Pods keep running and a new Instance simply stays pending. The InstanceType phase cannot drive this — it collapses both `Hold` and a fully-drained `HoldAndDrain` to `Inactive`, and a fast drain clears the reservation before a durable `Draining` phase is ever observed — so the stop keys on the `StopPolicy`. A `ClusterQueue` watch (on its `StopPolicy`) re-enqueues the type's Instances so the stop is prompt even when no Pod event fires; the `InstanceType` watch is narrowed to the deletion signal for a prompt teardown stop. diff --git a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/instanceresources.go b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/instanceresources.go index af2521f7..f71e40f4 100644 --- a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/instanceresources.go +++ b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/instanceresources.go @@ -25,7 +25,7 @@ type InstanceResourcesApplyConfiguration struct { // normalized .sliced.units; it is ignored by non-sliced requests. AcceleratorSlicedMemoryPercentage *int32 `json:"acceleratorSlicedMemoryPercentage,omitempty"` // AcceleratorSlicedCoresPercentage is the per-card compute (SM) budget requested on - // a sliced InstanceType, as a percentage in [0,100]. It must not be smaller than + // a sliced InstanceType, as a percentage in [0,100]. It is independent of // AcceleratorSlicedMemoryPercentage; when only one of the two is set the webhook // copies it to the other. It is ignored by non-sliced requests. AcceleratorSlicedCoresPercentage *int32 `json:"acceleratorSlicedCoresPercentage,omitempty"` diff --git a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/instancetypeunitresources.go b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/instancetypeunitresources.go index 8f43279b..f41732de 100644 --- a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/instancetypeunitresources.go +++ b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/instancetypeunitresources.go @@ -7,7 +7,7 @@ package v1alpha1 // // InstanceTypeUnitResources describes the unit resources of the InstanceType. type InstanceTypeUnitResourcesApplyConfiguration struct { - // CPU is the unit CPU resource(milli-cores) of the InstanceType. + // CPU is the unit CPU resource(cores) of the InstanceType. CPU *string `json:"cpu,omitempty"` // RAM is the unit RAM resource(Mi) of the InstanceType. RAM *string `json:"ram,omitempty"` diff --git a/pkg/worker/webhooks/worker/instance.go b/pkg/worker/webhooks/worker/instance.go index 185d4abb..ae9d2618 100644 --- a/pkg/worker/webhooks/worker/instance.go +++ b/pkg/worker/webhooks/worker/instance.go @@ -413,7 +413,7 @@ func validateResourceRequests(instType *worker.InstanceType, instRess *workercor switch { case instType.Spec.Sliceable: // The slice is requested as memory/compute percentages in [0,100] (0 disables - // slicing). The compute budget must not be smaller than the memory budget. + // slicing); the two budgets are independent. memPct := int64(instRess.AcceleratorSlicedMemoryPercentage) coresPct := int64(instRess.AcceleratorSlicedCoresPercentage) memPath := field.NewPath("spec.resources.acceleratorSlicedMemoryPercentage") @@ -426,10 +426,6 @@ func validateResourceRequests(instType *worker.InstanceType, instRess *workercor errs = append(errs, field.Invalid(coresPath, instRess.AcceleratorSlicedCoresPercentage, "must be between 0 and 100")) } - if coresPct > 0 && coresPct < memPct { - errs = append(errs, field.Invalid(coresPath, instRess.AcceleratorSlicedCoresPercentage, - "must not be less than the memory percentage")) - } if memPct > 0 || coresPct > 0 { // A sliced request (a non-zero percentage) is a fraction of ONE card: the // slice is expressed through the percentages, not the card count, so the diff --git a/pkg/worker/webhooks/worker/instance_test.go b/pkg/worker/webhooks/worker/instance_test.go index fbbb3894..61ebb582 100644 --- a/pkg/worker/webhooks/worker/instance_test.go +++ b/pkg/worker/webhooks/worker/instance_test.go @@ -81,8 +81,8 @@ func TestInstanceWebhook_ValidateCreate(t *testing.T) { } // TestInstanceWebhook_ValidateCreate_SlicedPercentages pins the sliced request -// contract: the memory/compute percentages must be in [0,100] and the compute -// budget may not be smaller than the memory budget. +// contract: the memory/compute percentages must be in [0,100] and are independent +// of each other. func TestInstanceWebhook_ValidateCreate_SlicedPercentages(t *testing.T) { const typeName = "sliced-8s" @@ -95,7 +95,7 @@ func TestInstanceWebhook_ValidateCreate_SlicedPercentages(t *testing.T) { {name: "compute larger than memory allowed", memPct: 20, coresPct: 50}, {name: "whole-card slice allowed", memPct: 100, coresPct: 100}, {name: "no slice allowed", memPct: 0, coresPct: 0}, - {name: "compute smaller than memory rejected", memPct: 50, coresPct: 20, wantErr: true}, + {name: "compute smaller than memory allowed", memPct: 50, coresPct: 20}, {name: "memory above 100 rejected", memPct: 101, coresPct: 101, wantErr: true}, {name: "compute above 100 rejected", memPct: 20, coresPct: 101, wantErr: true}, {name: "negative memory rejected", memPct: -1, wantErr: true}, diff --git a/pkg/worker/webhooks/worker/pod.go b/pkg/worker/webhooks/worker/pod.go index 213f1b8c..5669b22a 100644 --- a/pkg/worker/webhooks/worker/pod.go +++ b/pkg/worker/webhooks/worker/pod.go @@ -30,9 +30,9 @@ import ( // into the credit-counting .sliced.units so Kueue and the device-plugin agree — // any client-supplied .sliced.units is ignored and recomputed, since the value is // webhook-derived only. Validating: a .sliced request that omits the card count, -// names no memory, sets both memory keys, carries a non-positive or over-100 -// budget, or whose compute budget is smaller than its percentage memory budget is -// rejected. +// names no memory, sets both memory keys, carries a non-positive budget, or sets a +// percentage budget above 100 is rejected; a Pod mixing accelerator allocation +// modes (exclusive/shared/sliced) is also rejected. // // nolint: lll // +k8s:webhook-gen:validating:group="",version="v1",resource="pods",scope="Namespaced" @@ -300,9 +300,9 @@ func (r *PodWebhook) ValidateDelete(_ context.Context, _ runtime.Object) (ctrlad } // validateSlicedContainer checks one container's sliced request for the given -// manufacturer family: it must name exactly one positive memory budget, carry no -// non-positive sliced budget, and keep the compute budget at least as large as -// the percentage memory budget. +// manufacturer family: it must name exactly one memory budget (percentage or mib), +// every budget it sets must be positive, and the percentage budgets must not exceed +// 100. The compute and memory budgets are independent. func validateSlicedContainer(ctr *core.Container, names slicedResourceNames, idx int) field.ErrorList { var errs field.ErrorList path := field.NewPath("spec", "containers").Index(idx).Child("resources", "requests") @@ -340,10 +340,5 @@ func validateSlicedContainer(ctr *core.Container, names slicedResourceNames, idx "must be within (0, 100]")) } - if hasMemPct && memPctQ.Value() > 0 && hasCoresPct && coresPctQ.Value() < memPctQ.Value() { - errs = append(errs, field.Invalid(path.Key(string(names.coresPct)), coresPctQ.String(), - fmt.Sprintf("must not be less than %s", names.memPct))) - } - return errs } diff --git a/pkg/worker/webhooks/worker/pod_test.go b/pkg/worker/webhooks/worker/pod_test.go index 75e113d9..942db337 100644 --- a/pkg/worker/webhooks/worker/pod_test.go +++ b/pkg/worker/webhooks/worker/pod_test.go @@ -206,9 +206,8 @@ func TestPodWebhook_ValidateCreate(t *testing.T) { wantErr: true, }, { - name: "cores smaller than memory rejected", + name: "cores smaller than memory allowed", requests: map[core.ResourceName]string{names.card: "1", names.memPct: "50", names.coresPct: "20"}, - wantErr: true, }, { name: "non-positive memory-percentage rejected",