From a50d477f95c8709d560e0ccb56ba02370929fa74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Tue, 24 Feb 2026 09:06:40 +0100 Subject: [PATCH 01/21] docs: add AI coding agent developer guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add AGENTS.md with comprehensive developer guide for AI coding agents working on the Volcano codebase, covering build commands, test commands, code style, import conventions, and agent/batch scheduler architecture. Update .gitignore with local development tool paths. Signed-off-by: Hajnal Máté --- .gitignore | 9 + .opencode/skills/tilt-dev.md | 186 ++++++++++++++++ AGENTS.md | 412 +++++++++++++++++++++++++++++++++++ 3 files changed, 607 insertions(+) create mode 100644 .opencode/skills/tilt-dev.md create mode 100644 AGENTS.md diff --git a/.gitignore b/.gitignore index 5d8d076d170..237fc1f5d89 100644 --- a/.gitignore +++ b/.gitignore @@ -140,3 +140,12 @@ vendor # helm dependency files installer/helm/chart/volcano/requirements.lock + +#### Agentic entries +# Oh my opencode sisyphus agent work directory +# https://github.com/code-yeongyu/oh-my-opencode +.sisyphus/ +.sisyphus/* + +# Codex local environment metadata +.codex/ diff --git a/.opencode/skills/tilt-dev.md b/.opencode/skills/tilt-dev.md new file mode 100644 index 00000000000..e7c11a08a69 --- /dev/null +++ b/.opencode/skills/tilt-dev.md @@ -0,0 +1,186 @@ +# Tilt Dev Loop (Kind kind-volcano-dev) + +Use this skill to run and operate Volcano's local Tilt development loop on the +Kind cluster named `kind-volcano-dev`, provisioned by `ctlptl` with a local registry. + +## When to use + +- Start or stop local Tilt-driven development. +- Check resource health quickly. +- Fetch logs for failing resources. +- Trigger targeted rebuilds after code changes. +- Wait for the environment to become ready before tests. + +## Ground rules + +- Use repository root as the working directory. +- Always use Makefile wrappers (`make dev-*`) over raw `_output/bin/tilt` commands. +- Keep cluster name fixed to `kind-volcano-dev`. +- Use `hack/tilt/Tiltfile` as the single Tilt entrypoint. + +## Canonical workflow + +1) Start environment + +```bash +make dev-up +``` + +For long-running sessions, prefer daemon mode from the Makefile so logs do not +stream in the CLI: + +```bash +nohup make dev-up > /tmp/dev-up.log 2>&1 & +``` + +What this does: +- Ensures `tilt`, `kind`, and `ctlptl` binaries are available in `_output/bin/`. +- Creates/updates a `ctlptl`-managed Kind cluster + local registry from `hack/tilt/ctlptl-kind-registry.yaml`. +- Runs `tilt up -f hack/tilt/Tiltfile`. + +2) Quick status snapshot + +```bash +make dev-tilt-status +``` + +3) Wait for resources to settle + +```bash +make dev-wait-ready +``` + +Always run this after `make dev-up` before triggering tests. + +4) Inspect one resource deeply + +```bash +make dev-tilt-describe RESOURCE=volcano-scheduler +``` + +5) Tail logs + +All resources: + +```bash +make dev-tilt-logs +``` + +Single resource: + +```bash +make dev-tilt-logs RESOURCE=volcano-scheduler +``` + +Log hygiene for background mode: + +```bash +# clear previous daemon log before a fresh start +: > /tmp/dev-up.log +``` + +6) Force rebuild/redeploy for one resource + +```bash +make dev-tilt-trigger RESOURCE=volcano-scheduler +``` + +7) Stop environment + +```bash +make dev-down +``` + +8) Full cleanup (Tilt down + delete Kind cluster + remove local binaries) + +```bash +make dev-clean +``` + +## Troubleshooting playbook + +If `make dev-up` fails: + +1. Confirm cluster exists: + +```bash +_output/bin/ctlptl get cluster +``` + +Expected: a cluster named `kind-volcano-dev` appears in the list. + +2. Confirm context matches Tiltfile expectation: + +```bash +kubectl config get-contexts -o name | grep -E '^(kind-volcano-dev|kind-kind-volcano-dev)$' +``` + +3. Verify Tilt can see resources: + +```bash +make dev-tilt-status +``` + +4. If a resource is stuck/unhealthy, gather details and logs: + +```bash +make dev-tilt-describe RESOURCE= +make dev-tilt-logs RESOURCE= +``` + +5. Retry only the affected resource: + +```bash +make dev-tilt-trigger RESOURCE= +``` + +If an e2e test resource is stuck in `Pending` or `UpdatePending` state, it means +a dependency (e.g., `install-ginkgo`) has not been built yet. Dependencies only +need to be triggered once after Tilt comes up — subsequent test triggers reuse +the already-built dependency. + +Correct flow: + +1. Trigger the e2e test resource. +2. Check its state with `make dev-tilt-describe RESOURCE=`. +3. If it shows `Pending` / `UpdatePending`, trigger the missing dependency. +4. **Do NOT re-trigger the test resource** — Tilt already has it queued and will + run it automatically once the dependency becomes ready. +5. Just `make dev-wait-ready TILT_WAIT_RESOURCES=uiresource/`. + +Example: + +```bash +# 1. Trigger the test +make dev-tilt-trigger RESOURCE=e2e-tests-schedulingbase-focused-example + +# 2. Check state — if Pending, trigger the dependency +make dev-tilt-trigger RESOURCE=install-ginkgo + +# 3. Wait — Tilt runs the test automatically after the dependency completes +make dev-wait-ready TILT_WAIT_RESOURCES=uiresource/e2e-tests-schedulingbase-focused-example +``` + +## CI-style run + +Use non-interactive mode when a bounded run is needed: + +```bash +_output/bin/tilt ci -f hack/tilt/Tiltfile +``` + +## Notes for agentic usage + +- **Prefer use Makefile wrappers** over raw `_output/bin/tilt` commands: + `make dev-tilt-status`, `make dev-tilt-describe`, `make dev-tilt-logs`, + `make dev-tilt-trigger`, `make dev-wait-ready`, `make dev-up`, `make dev-down`. +- Keep commands deterministic and resource-scoped when possible. +- Collect status first, then logs, then trigger retries. +- Treat `make dev-up` as idempotent for local loops. +- First `make dev-up` run takes ~20-25 minutes (Go image compilation). Subsequent + runs reuse Docker layer cache and are much faster. +- To avoid full image rebuilds when switching branches, start Tilt on `master` + first, then `git checkout` to the feature branch. Tilt will perform an + incremental live-update instead of a full rebuild. +- Use `make dev-down` (keeps cluster) for quick restarts. Use `make dev-clean` + only for full teardown. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000..03c3b4efa9c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,412 @@ +# Volcano Developer Guide for AI Coding Agents + +This guide provides essential information for AI coding agents working on the Volcano codebase. + +## Project Overview + +Volcano is a Kubernetes-native batch scheduling system written in Go 1.25+. It extends kube-scheduler for AI/ML, Big Data, and HPC workloads. The project follows standard Go conventions with Kubernetes-specific patterns. + +## Build Commands + +### Core Binaries +```bash +# Build all components +make all + +# Build individual components +make vc-scheduler # Batch scheduler (job/podgroup-level) +make vc-agent-scheduler # Agent scheduler (pod-level) +make vc-controller-manager # Controller manager +make vc-webhook-manager # Webhook admission controller +make vc-agent # Agent node daemon (QoS, oversubscription) +make vcctl # CLI tool + +# Build command-line utilities +make command-lines # vcancel, vresume, vsuspend, vjobs, vqueues, vsub + +# Clean build artifacts +make clean +``` + +### Docker Images +```bash +# Build all images +make images + +# Build individual component images +make vc-scheduler-image +make vc-agent-scheduler-image +make vc-controller-manager-image +make vc-webhook-manager-image +make vc-agent-image +``` + +## Testing Commands + +### Unit Tests +```bash +# Run all unit tests +make unit-test + +# Run tests for specific package +go test -v volcano.sh/volcano/pkg/scheduler/cache + +# Run single test function +go test -v volcano.sh/volcano/pkg/scheduler/cache -run TestGetOrCreateJob + +# Run with race detector +go test -race volcano.sh/volcano/pkg/... + +# Clean test cache before running +go clean -testcache +``` + +### E2E Tests +```bash +# Run all e2e tests +make e2e + +# Run specific e2e test suites +make e2e-test-schedulingbase +make e2e-test-schedulingaction +make e2e-test-jobp +make e2e-test-jobseq +make e2e-test-vcctl +make e2e-test-stress +make e2e-test-cronjob +make e2e-test-admission-webhook +``` + +### Linting and Verification +```bash +# Run golangci-lint +make lint + +# Verify formatting and generated code +make verify + +# Verify generated YAML files +make verify-generated-yaml + +# Check licenses +make lint-licenses +``` + +## Code Style Guidelines + +### Import Organization + +Use **3-section imports** with blank lines between: + +```go +import ( + // Section 1: Standard library (alphabetically sorted) + "context" + "fmt" + "time" + + // Section 2: Third-party packages (k8s.io, github.com, etc.) + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + // Section 3: Local volcano.sh packages + batch "volcano.sh/apis/pkg/apis/batch/v1alpha1" + "volcano.sh/volcano/pkg/scheduler/api" +) +``` + +**Import prefix:** Use `goimports` with `local-prefixes: volcano.sh` (configured in .golangci.yml) + +### Naming Conventions + +- **Packages:** Short, lowercase, single word (e.g., `cache`, `scheduler`, `util`) +- **Exported types:** PascalCase (e.g., `SchedulerCache`, `NodeInfo`, `TaskInfo`) +- **Unexported types:** camelCase (e.g., `defaultEvictor`, `defaultBinder`) +- **Interfaces:** Descriptive names, often ending in -er (e.g., `Binder`, `Evictor`, `StatusUpdater`) +- **Functions:** PascalCase (exported) or camelCase (unexported), verb-noun pattern (e.g., `NewNodeInfo`, `AddTask`, `allocateIdleResource`) +- **Variables:** Short names in local scope (e.g., `pg`, `sc`, `ni`, `err`, `ctx`), descriptive names at package level +- **Test functions:** `Test__` or `Test` + +### Error Handling + +**Pattern A: Immediate return with context** +```go +if err != nil { + return fmt.Errorf("failed to find Job %v for Task %v", jobID, taskID) +} +``` + +**Pattern B: Logging with klog** +```go +if err != nil { + klog.Errorf("Failed to update pod <%v/%v> status: %v", pod.Namespace, pod.Name, err) + return err +} +``` + +**Pattern C: Error accumulation** +```go +if len(errs) != 0 { + return fmt.Errorf("failed to kill %d pods of %d", len(errs), total) +} +``` + +**Guidelines:** +- Use `fmt.Errorf` for error context (no error wrapping packages) +- Use `klog` for structured logging with verbosity levels (V(3), V(4), etc.) +- Check errors immediately: `if err != nil` +- Never ignore errors without explicit justification + +### Comments and Documentation + +**File headers:** Apache 2.0 license (see existing files) + +**Function documentation:** GoDoc style (starts with function name) +```go +// New returns a Cache implementation that manages scheduler state. +func New(config *rest.Config, ...) Cache { ... } + +// AddTask adds a task to the node and updates resource allocation. +func (ni *NodeInfo) AddTask(task *TaskInfo) error { ... } +``` + +**Struct documentation:** +```go +// NodeInfo is node-level aggregated information including resources, +// state, and scheduled tasks. +type NodeInfo struct { + Name string + State NodeState + // The releasing resource on that node + Releasing *Resource +} +``` + +**Inline comments:** Explain "why" not "what", use sparingly for complex logic + +### Formatting + +- **Indentation:** Tabs (not spaces) - enforced by `gofmt` +- **Line length:** No strict limit, but be reasonable (typically < 120 characters) +- **Blank lines:** Separate logical blocks, one blank line between functions +- **Whitespace:** Use `whitespace` linter settings (no trailing whitespace) + +### Test Patterns + +**Table-driven tests:** +```go +func TestKillJob(t *testing.T) { + testcases := []struct { + Name string + Job *v1alpha1.Job + ExpectVal error + }{ + { + Name: "KillJob success case", + Job: &v1alpha1.Job{...}, + ExpectVal: nil, + }, + } + for _, tc := range testcases { + t.Run(tc.Name, func(t *testing.T) { + // test logic + }) + } +} +``` + +**E2E tests:** Use Ginkgo/Gomega framework +```go +var _ = Describe("Job E2E Test", func() { + It("should run job successfully", func() { + Expect(err).NotTo(HaveOccurred()) + }) +}) +``` + +**Test helpers:** Use `build*` prefix for test object creation + +## Code Generation + +```bash +# Generate code (CRDs, clients, informers, listers) +make generate-code + +# Generate CRD manifests +make manifests + +# Generate YAML files +make generate-yaml + +# Generate Helm charts +make generate-charts +``` + +## Agent Scheduler (vc-agent-scheduler) + +The agent-scheduler is a pod-level scheduler introduced alongside the traditional batch scheduler. While `vc-scheduler` operates on jobs and podgroups (batch scheduling), `vc-agent-scheduler` schedules individual pods one at a time per cycle, similar to kube-scheduler but with Volcano's plugin architecture. + +### Architecture Comparison + +| Aspect | `vc-scheduler` (Batch) | `vc-agent-scheduler` (Pod-level) | +|---|---|---| +| Scheduling unit | Jobs / PodGroups | Individual pods | +| Package | `pkg/scheduler/` | `pkg/agentscheduler/` | +| Plugin interface | `scheduler/framework.Plugin` | `agentscheduler/framework.Plugin` | +| Action interface | `scheduler/framework.Action` | `agentscheduler/framework.Action` | +| Default actions | enqueue, allocate, preempt, reclaim, backfill | allocate | +| Default plugins | ~15+ (gang, proportion, predicates, nodeorder, etc.) | predicates, nodeorder | +| Unique concepts | Jobs, Queues, PodGroups, gang scheduling | Sharding (ShardCoordinator), Workers, SchedulingContext | + +### Plugin and Action Interfaces + +The agent-scheduler has its own plugin and action interfaces distinct from the batch scheduler. When writing agent-scheduler plugins, use the interfaces from `pkg/agentscheduler/framework/`: + +**Action interface:** +```go +type Action interface { + Name() string + OnActionInit(configurations []conf.Configuration) + Initialize() + Execute(fwk *Framework, schedCtx *agentapi.SchedulingContext) + UnInitialize() +} +``` + +**Plugin interface:** +```go +type Plugin interface { + Name() string + OnPluginInit(fwk *Framework) + OnCycleStart(fwk *Framework) + OnCycleEnd(fwk *Framework) +} +``` + +Key differences from the batch scheduler: +- Actions receive a `SchedulingContext` containing a single `TaskInfo` (one pod), not a full session +- Plugins use `OnPluginInit`/`OnCycleStart`/`OnCycleEnd` instead of `OnSessionOpen`/`OnSessionClose` +- The `BindContextHandler` interface allows plugins to inject bind-time extensions + +### Key Types + +```go +// SchedulingContext contains all information needed for scheduling a task +type SchedulingContext struct { + Task *api.TaskInfo + QueuedPodInfo *framework.QueuedPodInfo + NodesInShard sets.Set[string] +} + +// BindContext carries plugin-specific bind extensions +type BindContext struct { + SchedCtx *SchedulingContext + Extensions map[string]cache.BindContextExtension +} +``` + +### Default Configuration + +```yaml +actions: "allocate" +tiers: +- plugins: + - name: predicates + - name: nodeorder +``` + +### Sharding Model + +The agent-scheduler supports node sharding via `ShardCoordinator`, allowing multiple scheduler instances to partition nodes for horizontal scaling. This concept has no equivalent in the batch scheduler. + +- `ShardCoordinator` tracks which nodes belong to this scheduler instance +- Workers within a scheduler instance coordinate via revision-based node sets +- Sharding mode is configured via `--sharding-mode` (supports `hard` and `soft` modes) +- Node shard assignments are managed through `NodeShard` custom resources + +### Agent Node Daemon (vc-agent) + +The `vc-agent` binary (`pkg/agent/`) is a separate node-level daemon, unrelated to the agent-scheduler. It handles: +- **QoS management**: CPU throttling, CPU burst, memory QoS, CPU QoS, network QoS +- **Oversubscription**: Resource reporting and policy-based extend resources +- **Node monitoring**: Resource calculation, node health probes +- **Eviction**: Pod eviction based on node pressure + +The agent uses an event-driven architecture with probes (data sources) and handlers (actions): +- Probes: `pkg/agent/events/probes/` (pods, noderesources, nodemonitor) +- Handlers: `pkg/agent/events/handlers/` (cputhrottle, cpuburst, memoryqos, networkqos, eviction, etc.) +- Configuration: File-based or ConfigMap-based config sources (`pkg/agent/config/`) + +## Commit Message Format + +Follow this convention: +``` +: + + + +Fixes # +``` + +Example: +``` +scheduler: add gang scheduling support for Ray jobs + +This enables Ray jobs to use gang scheduling to ensure all workers +start together, improving resource utilization and reducing deadlocks. + +Fixes #1234 +``` + +**Guidelines:** +- Subject line ≤ 70 characters +- Body wrapped at 80 characters +- Focus on "why" not "what" + +## Linter Configuration + +Enabled linters (`.golangci.yml`): +- `gofmt`, `goimports`, `govet` - Go standard checks +- `staticcheck`, `gosimple`, `ineffassign` - Code quality +- `typecheck`, `unused` - Type safety and dead code +- `depguard` - Dependency restrictions (no `k8s.io/klog` v1, no `io/ioutil`) +- `whitespace` - Formatting + +**Deprecated packages to avoid:** +- `k8s.io/klog` → use `k8s.io/klog/v2` +- `io/ioutil` → use `io` and `os` packages (Go 1.16+) + +## Key Directories + +- `cmd/` - Main applications (scheduler, agent-scheduler, controller-manager, webhook-manager, agent, cli) +- `pkg/scheduler/` - Batch scheduler (job/podgroup-level scheduling) +- `pkg/agentscheduler/` - Agent scheduler (pod-level scheduling, sharding) +- `pkg/agent/` - Agent node daemon (QoS, oversubscription, eviction) +- `pkg/controllers/` - Controllers +- `pkg/webhooks/` - Webhooks +- `test/e2e/` - End-to-end tests +- `hack/` - Build and development scripts +- `installer/` - Deployment manifests and Dockerfiles +- `config/` - CRD definitions +- `staging/src/volcano.sh/apis/` - API definitions (local development) + +## Additional Resources + +- [Contributing Guide](contribute.md) +- [Development Setup](docs/development/prepare-for-development.md) +- [Build Instructions](docs/development/development.md) + +## Pull Request Follow-Up Workflow + +When a pull request comment mentions an AI coding agent for follow-up work, +use this lightweight workflow: + +1. Read the trigger comment first and identify the concrete request. +2. Review newer review comments and CI status to avoid repeating stale fixes. +3. Make only incremental changes on top of the PR branch. +4. Validate the changed scope with targeted commands before posting updates. +5. Summarize the change with clear verification results and next steps. + +This keeps follow-up iterations small, reviewable, and directly tied to the +latest reviewer feedback. From ab55011b24d116f8e70869c6bc724b93b5c7b82c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Tue, 24 Feb 2026 09:06:47 +0100 Subject: [PATCH 02/21] docs: add Tilt-based local development design proposal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add design document covering motivation, architecture, detailed design, tool comparison (Tilt vs Skaffold, DevSpace, Telepresence), and agentic workflow safety considerations for the Kind + ctlptl + Tilt local development workflow. Update prepare-for-development.md and development.md with Tilt-based quick start and full workflow reference. Signed-off-by: Hajnal Máté --- README.md | 19 +- docs/design/tilt-based-development.md | 535 ++++++++++++++++++++ docs/development/development.md | 142 ++++++ docs/development/prepare-for-development.md | 40 +- 4 files changed, 734 insertions(+), 2 deletions(-) create mode 100644 docs/design/tilt-based-development.md diff --git a/README.md b/README.md index 56745b30856..2a858b8bad2 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,23 @@ You can try Volcano by one of the following two ways. > * For Kubernetes v1.17 and above, use CRDs under config/crd/bases (recommended) > * For Kubernetes v1.16 and below, use CRDs under config/crd/v1beta1 (deprecated) +### Development with Tilt (Recommended for contributors) + +The fastest way to get a full development environment is with the Tilt-based workflow. +It provisions a Kind cluster with a local registry and deploys all Volcano components +with live-reload — edit any Go file and the running component rebuilds automatically: + +```bash +make dev-up # Provisions cluster + starts Tilt (first run ~20-25 min) +``` + +**Prerequisites:** Docker and Make. No local Go toolchain is required for building — builds +happen inside containers. However, installing Go locally is recommended for running unit +tests and working with the project long-term. + +For the full workflow reference, see the [development guide](docs/development/development.md). +For architecture and design rationale, see the [design proposal](docs/design/tilt-based-development.md). + ### Install with YAML files Install Volcano on an existing Kubernetes cluster. This way is both available for x86_64 and arm64 architecture. @@ -155,7 +172,7 @@ helm list -n volcano-system ### Install from code -If you don't have a kubernetes cluster, try one-click install from code base: +If you don't have a kubernetes cluster, and you don't want the live-reload tilt environment, you can try the legacy one-click install from code base: ```bash ./hack/local-up-volcano.sh diff --git a/docs/design/tilt-based-development.md b/docs/design/tilt-based-development.md new file mode 100644 index 00000000000..7c69a28a14f --- /dev/null +++ b/docs/design/tilt-based-development.md @@ -0,0 +1,535 @@ +# Tilt-Based Local Development Workflow + +## Motivation + +### Problem + +Developing and testing Volcano components locally requires building Docker images, pushing them to a registry, deploying to a Kubernetes cluster, and restarting pods for every code change. This cycle is slow and error-prone, creating friction for contributors who need fast feedback loops during scheduler, controller, or webhook development. + +The existing `hack/local-up-volcano.sh` script provides a one-shot local cluster setup, but it does not support iterative development. Each code change requires tearing down and rebuilding the environment, which can take several minutes. + +### Design Goal + +1. Enable sub-minute code-to-running-pod feedback for all Volcano control-plane components (scheduler, controller-manager, webhook-manager). +2. Provide a single-command workflow (`make dev-up`) that provisions a local Kind cluster with a container registry and deploys Volcano with live-reloading. +3. Support multi-architecture development (amd64 and arm64) without manual configuration. +4. Integrate unit tests and end-to-end tests as on-demand resources within the same development environment. +5. Keep all development tooling self-contained in `_output/bin` to avoid polluting the host system. + +## Architecture Overview + +The workflow is built on three core tools: + +- **[Kind](https://kind.sigs.k8s.io/)** — Local Kubernetes cluster running in Docker containers. +- **[ctlptl](https://github.com/tilt-dev/ctlptl)** — Declarative cluster lifecycle manager that provisions Kind clusters with an attached local container registry. +- **[Tilt](https://tilt.dev/)** — Development environment that watches source files, builds container images, and live-updates running pods. + +``` +Developer workstation ++-------------------------------------------------------------------+ +| | +| make dev-up | +| | | +| +-> dev-install-tilt (download Tilt binary to _output/bin) | +| +-> dev-install-kind (download Kind binary to _output/bin) | +| +-> dev-install-ctlptl (download ctlptl binary to _output/bin)| +| +-> dev-create-kind-cluster (ctlptl apply) | +| +-> tilt up | +| | ++-------------------------------------------------------------------+ +| | +| Kind cluster: kind-volcano-dev | +| +-------------------------------------------------------------+ | +| | volcano-system namespace | | +| | +------------------+ +------------------+ | | +| | | vc-scheduler | | vc-controller- | | | +| | | (live-reload) | | manager | | | +| | +------------------+ | (live-reload) | | | +| | +------------------+ | | +| | +------------------+ +------------------+ | | +| | | vc-webhook- | | admission-init | | | +| | | manager | | (Job) | | | +| | | (live-reload) | +------------------+ | | +| | +------------------+ | | +| | | | +| | volcano-monitoring namespace | | +| | +------------------+ +-------+ +------------------+ | | +| | | prometheus | |grafana| | kube-state- | | | +| | | (:3001) | |(:3000)| | metrics | | | +| | +------------------+ +-------+ +------------------+ | | +| +-------------------------------------------------------------+ | +| | +| Local registry: volcano-registry:5005 | ++-------------------------------------------------------------------+ +``` + +### Component Relationship + +1. **ctlptl** provisions a Kind cluster (`kind-volcano-dev`) with 1 control-plane node and 4 worker nodes, plus a local Docker registry (`volcano-registry:5005`). +2. **Tilt** watches the Volcano source tree and uses `docker_build` with `live_update` to sync code changes into running containers. +3. Inside each container, an **entrypoint.sh** script runs the Volcano binary as a child process. When Tilt syncs new files and touches `/tmp/reload`, the script stops the process, rebuilds the binary in-container, and restarts it. +4. The Helm chart is rendered by Tilt's `helm()` function with development-specific value overrides (increased log verbosity, relaxed scheduling period, control-plane tolerations). + +## File Layout + +All Tilt-specific files live under `hack/tilt/`: + +``` +hack/tilt/ ++-- Makefile # Make targets for dev lifecycle ++-- Tiltfile # Tilt orchestration (builds, deploys, resources) ++-- Dockerfile.tilt # Multi-stage Dockerfile for scheduler/controller/webhook ++-- Dockerfile.admission-init.tilt # Lightweight Dockerfile for admission init Job ++-- entrypoint.sh # Live-reload entrypoint for Volcano binaries ++-- ctlptl-kind-registry.yaml # ctlptl declarative config for Kind + registry ++-- values-tilt-override.yaml # Helm value overrides for local development +``` + +The `hack/tilt/Makefile` is included by the root `Makefile`, exposing `dev-*` targets at the top level. + +## Detailed Design + +### Cluster Provisioning + +The cluster is managed declaratively through ctlptl. The configuration is composed at apply time from two files: + +1. `ctlptl-kind-registry.yaml` — defines the local Docker registry and the cluster identity: + +```yaml +apiVersion: ctlptl.dev/v1alpha1 +kind: Registry +name: volcano-registry +port: 5005 +--- +apiVersion: ctlptl.dev/v1alpha1 +kind: Cluster +name: kind-volcano-dev +product: kind +registry: volcano-registry +``` + +2. `hack/e2e-kind-config.yaml` — the shared Kind cluster configuration (feature gates, containerd patches, node topology, kubeadm patches). This file is the single source of truth for Kind cluster settings, shared between the Tilt dev workflow and the e2e test workflow (`run-e2e-kind.sh`). + +At `make dev-create-kind-cluster` time, the Makefile merges these two files: it appends `e2e-kind-config.yaml` content (stripped of `kind:` and `apiVersion:` headers) under the `kindV1Alpha4Cluster:` key and pipes the result to `ctlptl apply -f -`. This ensures any Kind cluster config changes (e.g., new feature gates, node count, kubeadm patches) are automatically picked up by both workflows without duplication. + +ctlptl handles: +- Creating the local Docker registry container. +- Creating the Kind cluster with the registry pre-configured so that images pushed to `localhost:5005` are accessible inside the cluster. +- Idempotent applies — running `ctlptl apply` when the cluster already exists is a no-op. + +### Image Building and Live-Reload + +Tilt builds four container images: + +| Image | Dockerfile | Live-reload | Purpose | +|---|---|---|---| +| `volcanosh/vc-scheduler` | `Dockerfile.tilt` | Yes | Batch scheduler | +| `volcanosh/vc-controller-manager` | `Dockerfile.tilt` | Yes | Controller manager | +| `volcanosh/vc-webhook-manager` | `Dockerfile.tilt` | Yes | Webhook admission controller | +| `volcanosh/vc-admission-init` | `Dockerfile.admission-init.tilt` | No | One-shot TLS certificate init Job | + +The three main components share a single `Dockerfile.tilt` parameterized by `COMPONENT` build arg. The Dockerfile: + +1. Starts from `golang:1.25.0` (matching the project's Go version). +2. Copies source code and runs `go mod download`. +3. Builds the binary with `GOOS=linux GOARCH=${TARGETARCH}`. +4. Uses `dumb-init` as PID 1 to forward signals correctly. +5. Delegates to `entrypoint.sh` for runtime process management. + +The `live_update` configuration syncs `pkg/`, `cmd/`, `third_party/`, and `entrypoint.sh` into the running container and touches `/tmp/reload` to trigger a rebuild. + +#### Live-Reload Mechanism + +The `entrypoint.sh` script implements a file-watch loop: + +``` +1. Build and start the Volcano binary as a background child process. +2. Loop every 1 second: + a. Check if /tmp/reload exists. + b. If yes: stop child, rebuild binary (go build), start child, remove /tmp/reload. +``` + +The rebuild uses `GOARCH="$(go env GOARCH)"` to detect the container's architecture at runtime, supporting both amd64 and arm64 Kind clusters. + +### Multi-Architecture Support + +All download URLs and build commands use architecture-aware variables: + +- **Tilt and ctlptl downloads** (`hack/tilt/Makefile`): A `HOSTARCH` variable maps `uname -m` output to release archive naming (`x86_64` stays as-is, `aarch64` becomes `arm64`). +- **Docker image builds** (`Dockerfile.tilt`): Uses `ARG TARGETARCH` (set automatically by Docker BuildKit) for cross-compilation. +- **In-container rebuilds** (`entrypoint.sh`): Uses `go env GOARCH` to detect the runtime architecture. + +### Helm Integration + +Tilt renders the Volcano Helm chart using its built-in `helm()` function, combining: + +1. The default `values.yaml` from the chart. +2. A `values-tilt-override.yaml` with development-specific settings: + - `image_pull_policy: IfNotPresent` (images come from the local registry). + - Scheduler and controller log verbosity at level 5. + - Scheduling period reduced to 1 second. + - Control-plane tolerations on all components (so pods can schedule on Kind's control-plane node if needed). + +### Resource Grouping + +Tilt resources are organized into labeled groups for the Tilt UI: + +- **volcano**: Core Volcano resources (namespaces, CRDs, admission, scheduler, controllers, roles). +- **monitoring**: Prometheus, Grafana, kube-state-metrics. +- **tests-install**: Manual install targets for test dependencies (KWOK, Ginkgo). +- **tests**: Unit tests and per-suite e2e test triggers (all manual, on-demand). + +Resource dependencies ensure correct ordering: +- `volcano-admission` depends on `volcano-namespaces`, `volcano-crds`, and `volcano-admission-init`. +- `volcano-scheduler` and `volcano-controllers` depend on `volcano-namespaces`. + +### Testing Integration + +The Tiltfile registers test suites as `local_resource` entries with `trigger_mode=TRIGGER_MODE_MANUAL` and `auto_init=False`: + +- `unit-tests` - Runs `make unit-test`. +- `e2e-tests-all` - Runs all e2e suites sequentially. +- Individual suite targets: `e2e-tests-jobp`, `e2e-tests-jobseq`, `e2e-tests-schedulingbase`, `e2e-tests-schedulingaction`, `e2e-tests-vcctl`, `e2e-tests-dra`, `e2e-tests-hypernode`, `e2e-tests-cronjob`. +- `e2e-tests-schedulingbase-focused-example` — A pre-configured single-test example (see [Focused Test Execution](#focused-test-execution) below). + +Test dependencies (KWOK, Ginkgo) are registered as separate manual install resources under the `tests-install` label. E2E test resources declare `resource_deps` on `install-ginkgo` (and `install-kwok` for `e2e-tests-all`), which means Tilt will block a triggered test from running until its dependencies have reached the Ready state. However, since the dependencies are also manual-trigger resources, they are **not** triggered automatically — the developer must trigger them explicitly on first use. Once a dependency has been triggered and reaches Ready, it stays Ready for the remainder of the Tilt session, so subsequent test runs only require triggering the test resource itself. + +The typical first-run workflow is: + +1. Trigger the test resource (e.g., `make dev-tilt-trigger RESOURCE=e2e-tests-schedulingbase`). +2. The test enters `Pending` state because `install-ginkgo` is not yet Ready. +3. Trigger the dependency (`make dev-tilt-trigger RESOURCE=install-ginkgo`). +4. Once the dependency completes, the test unblocks and runs automatically. +5. On subsequent runs, only step 1 is needed — the dependency is already Ready. + +#### E2E Test Runner Integration + +Running Ginkgo E2E tests during local development is traditionally cumbersome: the CI script `hack/run-e2e-kind.sh` handles cluster creation, KWOK setup, Volcano installation, and Ginkgo invocations as a single pipeline, making it unusable against an already-running Tilt cluster. Developers would otherwise need to construct manual `ginkgo` commands with the correct suite paths, `--focus`/`--skip` flags, and `KUBECONFIG` plumbing. + +The Tilt integration solves this through three mechanisms: + +**1. `run-ginkgo-suite()` — centralized Ginkgo invocation.** + +A function in `hack/run-e2e-kind.sh` centralizes all Ginkgo flag construction: + +```bash +run-ginkgo-suite [ginkgo_flags...] +``` + +Every test suite (in both CI and Tilt) calls this function instead of invoking `ginkgo` directly. The function handles `--focus` and `--skip` flag assembly, ensuring consistent behavior across all execution contexts. Additional ginkgo flags (e.g., `--slow-spec-threshold`, `-p`) are passed through as variadic arguments. + +**2. `E2E_LOCAL_ONLY=1` — skip bootstrapping for Tilt-managed clusters.** + +When `E2E_LOCAL_ONLY=1` is set, `hack/run-e2e-kind.sh` skips: +- Kind cluster creation (`kind-up-cluster`) +- KWOK installation (`install-kwok-with-helm`) +- Volcano Helm installation (`install-volcano`) +- Ginkgo installation (`install-ginkgo-if-not-exist` — assumed pre-installed via Tilt's `install-ginkgo` resource) + +The script proceeds directly to running the requested test suite against the existing cluster. This reduces test invocation from minutes to seconds. + +**3. `E2E_FOCUS` and `E2E_SKIP` — runtime overrides for test selection.** + +Two environment variables override the per-suite default focus and skip patterns: + +```bash +# Run only "Gang scheduling" tests, skipping sig-tagged and Full Occupied tests +E2E_FOCUS="Gang scheduling" E2E_SKIP="\[sig-.*\]|Full Occupied" \ + E2E_LOCAL_ONLY=1 E2E_TYPE=SCHEDULINGBASE hack/run-e2e-kind.sh +``` + +When `E2E_FOCUS` or `E2E_SKIP` is set, it takes precedence over the suite's hardcoded defaults. When unset, the defaults apply unchanged. This lets developers narrow test scope without editing any files. + +##### Focused test execution + +The Tiltfile includes a pre-configured example resource for running a single test: + +``` +e2e-tests-schedulingbase-focused-example + E2E_FOCUS = "Gang scheduling" (default, overridable via env) + E2E_SKIP = "\[sig-.*\]|Full Occupied" (default, overridable via env) + Suite = SCHEDULINGBASE +``` + +This resource demonstrates single-test execution within Tilt. Developers iterating on a specific feature can: + +1. **Copy this resource pattern** in the Tiltfile, changing the `E2E_FOCUS` default to match their test. +2. **Override at runtime** by setting `E2E_FOCUS` and `E2E_SKIP` environment variables before starting Tilt (they are passed through to `hack/run-e2e-kind.sh`). +3. **Trigger from the Tilt UI or CLI** — `make dev-tilt-trigger RESOURCE=e2e-tests-schedulingbase-focused-example` — getting results for a single test in seconds rather than waiting for the entire suite. + +This is particularly valuable for scheduler and controller development, where a full E2E suite run takes 10-30 minutes but a single focused test completes in under a minute. + +##### CI and Tilt alignment + +Both CI and Tilt call the same `hack/run-e2e-kind.sh` script with the same `run-ginkgo-suite()` function. The only difference is the environment: + +| Aspect | CI (`run-e2e-kind.sh`) | Tilt (`E2E_LOCAL_ONLY=1`) | +|---|---|---| +| Cluster creation | `kind-up-cluster` | Skipped (cluster exists) | +| Volcano install | `install-volcano` | Skipped (Tilt manages it) | +| Ginkgo install | `install-ginkgo-if-not-exist` | Skipped (Tilt `install-ginkgo` resource) | +| Test invocation | `run-ginkgo-suite()` | `run-ginkgo-suite()` (identical) | +| Focus/skip | Per-suite defaults | Defaults or `E2E_FOCUS`/`E2E_SKIP` overrides | + +This single-source-of-truth approach ensures that a test passing locally in Tilt will also pass in CI (and vice versa). + +### Version Management + +Tool versions follow a single-source-of-truth principle: + +| Tool | Version Variable | Defined In | +|---|---|---| +| Kind | `KIND_VERSION` | Root `Makefile` | +| Tilt | `TILT_VERSION` | `hack/tilt/Makefile` | +| ctlptl | `CTLPTL_VERSION` | `hack/tilt/Makefile` | + +The root `Makefile` already defines `KIND_VERSION` for CI usage. The Tilt Makefile requires it as a prerequisite (`ifndef KIND_VERSION ... $(error ...)`), ensuring consistency between CI and local development. + +## Usage + +### Quick Start + +```bash +# Start the development environment (installs tools, creates cluster, starts Tilt) +make dev-up + +# In another terminal, check resource status +make dev-tilt-status + +# View logs for a specific component +make dev-tilt-logs RESOURCE=volcano-scheduler + +# Wait for all resources to be ready +make dev-wait-ready +``` + +### Development Workflow + +1. Run `make dev-up` to start Tilt. +2. Edit any Go source file under `pkg/`, `cmd/`, or `third_party/`. +3. Tilt automatically syncs the changed files into the running container. +4. The entrypoint script detects the sync, rebuilds the binary, and restarts the process. +5. Check logs in the Tilt UI (http://localhost:10350) or via `make dev-tilt-logs`. + +### Running Tests + +Test dependencies (`install-ginkgo`, `install-kwok`) must be triggered once per Tilt session before running e2e tests. After the first trigger they stay Ready for subsequent runs. + +```bash +# First time: trigger test dependency installation (once per session) +make dev-tilt-trigger RESOURCE=install-ginkgo + +# Run unit tests (no dependencies needed) +make dev-tilt-trigger RESOURCE=unit-tests + +# Run a specific e2e suite +make dev-tilt-trigger RESOURCE=e2e-tests-schedulingbase + +# If the test stays in Pending, trigger its dependency first +# (install-ginkgo for most suites, install-kwok additionally for e2e-tests-all) + +# Run a focused single test (uses E2E_FOCUS/E2E_SKIP env vars) +make dev-tilt-trigger RESOURCE=e2e-tests-schedulingbase-focused-example + +# Check test results +make dev-tilt-logs RESOURCE=e2e-tests-schedulingbase +``` + +### Teardown + +```bash +# Stop Tilt but keep the cluster +make dev-down + +# Full cleanup: stop Tilt, delete cluster, remove tool binaries +make dev-clean +``` + +### Available Make Targets + +| Target | Description | +|---|---| +| `dev-up` | Install tools, create cluster, start Tilt | +| `dev-down` | Stop Tilt, leave cluster running | +| `dev-clean` | Stop Tilt, delete cluster and registry, remove tool binaries | +| `dev-status` | Show cluster and connection info | +| `dev-tilt-status` | Show status of all Tilt-managed resources | +| `dev-tilt-logs` | Show logs (all or `RESOURCE=`) | +| `dev-tilt-describe` | Describe a Tilt resource (`RESOURCE=`) | +| `dev-tilt-trigger` | Manually trigger a resource (`RESOURCE=`) | +| `dev-wait-ready` | Block until all resources reach Ready state | +| `dev-install-kind` | Install Kind binary to `_output/bin` | +| `dev-install-tilt` | Install Tilt binary to `_output/bin` | +| `dev-install-ctlptl` | Install ctlptl binary to `_output/bin` | +| `dev-create-kind-cluster` | Create Kind cluster and registry via ctlptl | +| `dev-delete-kind-cluster` | Delete Kind cluster and registry | + +## Design Decisions + +### Why Kind over k3d + +Kind (Kubernetes IN Docker) was chosen over k3d for the following reasons: + +- Kind is the de facto standard for local Kubernetes development in the Kubernetes ecosystem. +- Volcano's CI already uses Kind for e2e testing, ensuring parity between local development and CI. +- Kind uses `kubeadm`-bootstrapped clusters with unmodified upstream Kubernetes, giving higher fidelity for testing scheduler behavior. +- k3d uses k3s (a trimmed-down distribution) which may mask issues related to full Kubernetes API behavior. + +### Why ctlptl over raw Kind commands + +ctlptl provides declarative cluster management with built-in local registry support: + +- A single YAML file defines both the cluster and its registry, replacing multi-step imperative setup scripts. +- Idempotent operations: `ctlptl apply` only creates resources that do not already exist. +- Registry wiring is automatic — ctlptl configures Kind nodes to trust the local registry without manual containerd config patches. +- Cascade deletion (`--cascade=true`) ensures clean teardown of both cluster and registry. + +### Why Tilt over Skaffold and Other Tools + +Several tools exist for Kubernetes-native development workflows. This section compares +the options considered and explains why Tilt is the best fit for Volcano. + +#### Comparison Matrix + +| Capability | [Tilt](https://tilt.dev/) | [Skaffold](https://skaffold.dev/) | [DevSpace](https://devspace.sh/) | [Telepresence](https://www.telepresence.io/) | +|---|---|---|---|---| +| File-level live sync into running containers | Yes (`live_update`) | Partial (`sync` in v2, limited) | Yes (`sync`) | N/A (intercept-based) | +| In-container rebuild on sync | Yes (via custom entrypoint) | No (rebuilds image or restarts pod) | Yes (via hooks) | N/A | +| Native Helm chart rendering | Yes (`helm()` built-in) | Yes (`helm` deployer) | Yes (helm integration) | No | +| Resource dependency DAG | Yes (explicit `resource_deps`) | No | No | No | +| Web UI with live status | Yes (built-in at :10350) | No (CLI only) | Yes (optional UI) | No | +| Manual-trigger resources | Yes (`TRIGGER_MODE_MANUAL`) | No | No | No | +| Context safety guard | Yes (`allow_k8s_contexts`) | Yes (`kubeContext`) | Yes (`vars.DEVSPACE_CONTEXT`) | Partial | +| Starlark scripting | Yes (full Starlark) | No (YAML only) | No (YAML + hooks) | No | +| Active maintenance (2025) | Yes | Maintenance mode | Yes | Yes | + +#### Why Tilt wins for Volcano + +**1. File-granularity live sync without image rebuilds.** +Volcano's Go source tree is large (~500k+ lines across `pkg/`, `cmd/`, `staging/`). +A full Docker image rebuild on every change takes 30-90 seconds even with layer caching. +Tilt's `live_update` syncs only the changed `.go` files into the running container and +triggers an in-container `go build`, bringing incremental feedback down to 5-15 seconds. +Skaffold's sync support is limited to interpreted languages (Python, Node.js) — it +cannot trigger a recompilation step after syncing Go source files, so it falls back to +full image rebuilds for compiled languages. + +**2. Resource dependency DAG matches Volcano's deployment ordering.** +Volcano has strict deployment dependencies: CRDs must exist before admission webhooks +register, the admission-init Job must complete before the admission deployment starts, +and namespaces must exist before anything else. Tilt's `resource_deps` models this +DAG explicitly. Skaffold deploys manifests in a flat sequence and has no built-in +dependency mechanism, requiring workarounds like init containers or manual ordering. + +**3. Manual-trigger test resources.** +The Tiltfile registers unit tests and eight separate e2e test suites as on-demand +resources. Developers can trigger them from the Tilt UI or CLI without leaving the +development loop. Skaffold has no concept of manual-trigger resources — everything +in the pipeline runs automatically or not at all. + +**4. Starlark scripting for complex orchestration.** +The Tiltfile uses Starlark (a Python-like language) to loop over components, share +Dockerfiles via build args, conditionally include monitoring resources, and compose +Helm values. Skaffold's YAML-only configuration requires verbose repetition for +parameterized builds and cannot express conditional logic. + +**5. Skaffold's uncertain future.** +Google announced in late 2023 that Skaffold is in maintenance mode. While it remains +functional, new feature development has stopped. Tilt continues active development +under Docker, Inc. + +#### Why not DevSpace? + +DevSpace offers comparable sync and in-container build capabilities. However: + +- DevSpace lacks a resource dependency DAG, which Volcano's deployment ordering requires. +- DevSpace's Helm integration is functional but less mature than Tilt's native `helm()` function, which handles value layering and set overrides naturally. +- DevSpace's configuration format (devspace.yaml) is less expressive than Starlark for looping over multiple components with shared build logic. +- Tilt has stronger adoption in the Kubernetes ecosystem (used by Cluster API, Crossplane, Cilium, and similar infrastructure projects), providing better community support and proven patterns for controller/operator development. + +#### Why not Telepresence? + +Telepresence takes a fundamentally different approach: instead of syncing code into +cluster containers, it intercepts traffic from the cluster and routes it to a locally +running process. This is powerful for debugging a single service but is a poor fit +for Volcano because: + +- Volcano has three tightly coupled control-plane components (scheduler, controller-manager, webhook-manager) that need to run simultaneously. Telepresence intercepts work per-service, not per-cluster. +- Scheduler behavior depends on cluster state (nodes, pods, queues) that is difficult to replicate locally outside the cluster. +- The webhook-manager requires in-cluster TLS certificates generated by the admission-init Job, which cannot work with a local process. + +### Why Tilt is ideal for Volcano specifically + +Beyond the general tool comparison, Tilt aligns with Volcano's development model +in ways that are specific to Kubernetes scheduler/controller projects: + +- **Multi-component coordination.** Volcano is not a single binary — it is three cooperating control-plane processes plus an admission init Job. Tilt's resource model treats each as a first-class resource with health checks, logs, and dependency ordering, giving developers visibility into the full system rather than one component at a time. +- **Scheduler testing requires real cluster state.** Volcano's scheduler makes decisions based on node resources, pod groups, queue configurations, and CRD state. Unlike a web application where you can mock backends, meaningful scheduler development requires a running cluster with actual Kubernetes objects. Tilt's Kind integration provides this with zero manual setup. +- **Helm chart is the source of truth.** Volcano is deployed via Helm in production. By rendering the same Helm chart for local development (with override values), Tilt ensures that template changes, RBAC rules, and ConfigMap updates are tested in the development loop rather than discovered at deploy time. +- **CI parity.** Volcano's CI already uses Kind clusters. Tilt reuses the same Kind version and cluster configuration, so issues caught locally reproduce in CI and vice versa. + +### Why in-container builds + +The `Dockerfile.tilt` runs builds inside the container rather than on the host: + +- Eliminates the need for a local Go toolchain on the developer's machine. +- Ensures the build environment matches the container's OS and architecture exactly. +- `go mod download` is cached in the Docker layer, so only changed source files trigger recompilation. +- Live-update syncs source files and triggers `go build` inside the container, achieving sub-minute rebuild times for incremental changes. + +## Limitations and Future Work + +- **Linux-only tool downloads**: The Makefile downloads Linux-specific Tilt and ctlptl binaries. macOS and Windows developers would need to install these tools separately or extend the download logic. +- **No GPU support**: The Kind cluster does not configure GPU passthrough. GPU-dependent scheduling features cannot be tested locally. +- **Single cluster topology**: The current setup uses a fixed 1 control-plane + 4 worker node topology. A future enhancement could make node count configurable. +- **No agent-scheduler or vc-agent support**: Only the three core control-plane components are included. Adding agent-scheduler and vc-agent images would extend coverage to the full Volcano stack. + +### Agentic Development Workflows + +The Tilt-based workflow has an additional benefit for AI coding agent workflows. +When AI agents assist with Volcano development, the development environment's +safety boundaries become critical. + +#### The kubectl and Helm risk for agents + +AI coding agents that interact with Kubernetes clusters directly via `kubectl` and +`helm` face significant risks: + +- **Destructive operations.** A single `kubectl delete` or `helm uninstall` can destroy + cluster state. An agent that misinterprets a prompt could delete CRDs, wipe namespaces, + or uninstall the system it is developing against. +- **Stateful side effects.** Unlike file edits (which can be reverted via `git checkout`), + cluster mutations are stateful and often irreversible. Deleted PersistentVolumeClaims, + evicted pods, and removed finalizers cannot be undone. +- **Credential exposure.** Agents with `kubectl` access inherit the user's kubeconfig, + which may contain credentials for production clusters. A context switch error could + direct destructive commands at the wrong cluster. +- **Unbounded blast radius.** `kubectl apply -f` on a malformed manifest, `helm upgrade` + with incorrect values, or `kubectl patch` with a bad JSON path can cascade into + cluster-wide failures. + +#### How Tilt mitigates these risks + +The Tilt-based workflow provides a safety layer that makes agent-assisted development +viable: + +- **`allow_k8s_contexts` as a hard guard.** The Tiltfile explicitly lists allowed + Kubernetes contexts (`kind-volcano-dev`). Tilt refuses to operate against any other + context, making it impossible to accidentally target a production cluster. +- **Declarative-only mutations.** All cluster changes flow through Tilt's declarative + resource model. There is no need for agents to run `kubectl apply` or `helm install` + directly — Tilt handles deployment as a side effect of source file changes. +- **Make targets as the agent interface.** Agents interact with the development + environment through Make targets (`dev-up`, `dev-down`, `dev-tilt-status`, + `dev-tilt-logs`), which are safe, idempotent, and well-scoped. This eliminates + the need to grant agents direct `kubectl` or `helm` access. +- **Reproducible teardown.** If an agent corrupts the cluster state, `make dev-clean` + destroys the entire Kind cluster and registry, and `make dev-up` recreates it from + scratch in minutes. The blast radius is bounded to a disposable local cluster. +- **Source-driven feedback loop.** Agents edit Go source files, Tilt syncs and rebuilds. + The agent never needs to reason about Kubernetes resource lifecycles, manifest + ordering, or Helm release state. The complexity is encapsulated in the Tiltfile. + +This design means an AI coding agent can safely develop Volcano components by editing +source files and reading logs, without ever needing cluster-admin privileges or direct +access to Kubernetes APIs. diff --git a/docs/development/development.md b/docs/development/development.md index 48d1559465a..e5eca24aa91 100644 --- a/docs/development/development.md +++ b/docs/development/development.md @@ -1,5 +1,6 @@ This document helps you get started using the Volcano code base. If you follow this guide and find a problem, please take a few minutes to update this file. +- [Tilt-based local development](#tilt-based-local-development) - [Building the code](#building-the-code) - [Building docker images](#building-docker-images) - [Building a specific docker image](#building-a-specific-docker-image) @@ -164,4 +165,145 @@ Before sending pull requests, you should at least make sure your changes have pa - The preferred method of testing multiple scenarios or input is [table-driven testing](https://go.dev/wiki/TableDrivenTests). - Concurrent unit test runs must pass. +## Tilt-Based Local Development + +For iterative development with live-reloading, Volcano provides a Tilt-based workflow +that eliminates the manual build-push-deploy cycle. One command sets up everything: + +```bash +make dev-up +``` + +This installs Kind, Tilt, and ctlptl into `_output/bin`, creates a local Kind cluster +with a Docker registry, deploys Volcano via Helm, and starts Tilt for live-reloading. +No local Go toolchain is required — all builds happen inside containers. + +After `dev-up` completes, edit any Go source file under `pkg/`, `cmd/`, or +`third_party/`. Tilt syncs the changes into the running container and the entrypoint +script rebuilds the binary and restarts the process automatically. + + +### Tips and Best Practices + +**First-run startup time.** The initial `make dev-up` takes roughly 20-25 minutes +because Docker builds the Go compiler image layers and compiles all Volcano binaries +from scratch. The next builds will be somewhat faster due to the cached layers. + +**Avoid unnecessary image rebuilds across branches.** Tilt rebuilds container images +whenever the tracked source files differs from the last build **at startup**. If you switch branches +frequently, you can avoid long rebuilds by keeping the cluster and Tilt running on +your base branch (e.g. `master`) and only switching branches _after_ the initial +build completes: + +1. Start from a clean state on your base branch: + ```bash + git checkout master + make dev-up # builds images and caches layers + ``` +2. Switch to your feature branch while Tilt is running: + ```bash + git checkout my-feature-branch + ``` +3. Tilt detects the changed files and performs an incremental live-update (syncing + only the changed sources into the running container) instead of a full image + rebuild. This brings branch-switch turnaround down to seconds. +4. After you finished the work save it and bring down the tilt environment. + ```bash + git commit -m "Let's save this work" + make dev-down + ``` +5. **BEFORE** you continue the work go back to master. + If you stay on the `my-feature-branch` tilt will do a full ~20-25 minute rebuild. + ```bash + git checkout master + make dev-up + ``` +6. Go back to feature branch if tilt is running. (no rebuild) + ```bash + git checkout my-feature-branch + ``` + +**Keep `dev-down` vs `dev-clean` straight.** Use `make dev-down` to stop Tilt while +keeping the Kind cluster alive — this lets you inspect the cluster with `kubectl` +and restart Tilt later without re-creating the cluster. Use `make dev-clean` only +when you want a full teardown. + +### Viewing Status and Logs + +```bash +# Open the Tilt web UI +# (automatically available at http://localhost:10350 after dev-up) + +# Check resource status from the terminal +make dev-tilt-status + +# View logs for a specific component +make dev-tilt-logs RESOURCE=volcano-scheduler + +# Describe a resource in detail +make dev-tilt-describe RESOURCE=volcano-scheduler + +# Wait until all resources are ready +make dev-wait-ready +``` + +### Running Tests via Tilt + +Test resources are registered in Tilt with manual triggers so they do not run +automatically. Test dependencies (`install-ginkgo`, `install-kwok`) must be +triggered once per Tilt session — after the first trigger they stay Ready for all +subsequent test runs. + +```bash +# First time only: install test dependencies (once per Tilt session) +make dev-tilt-trigger RESOURCE=install-ginkgo +# install-kwok is additionally needed for e2e-tests-all + +# Run unit tests (no dependencies needed) +make dev-tilt-trigger RESOURCE=unit-tests + +# Run a specific e2e suite +make dev-tilt-trigger RESOURCE=e2e-tests-schedulingbase + +# If the test stays in Pending state, its dependency (install-ginkgo) has +# not been triggered yet — trigger it and the test will unblock automatically + +# Run a focused single test +make dev-tilt-trigger RESOURCE=e2e-tests-schedulingbase-focused-example + +# Check test output +make dev-tilt-logs RESOURCE=e2e-tests-schedulingbase +``` + +### Teardown + +```bash +# Stop Tilt but keep the Kind cluster running (for manual kubectl inspection) +make dev-down + +# Full cleanup: stop Tilt, delete cluster and registry, remove tool binaries +make dev-clean +``` + +### All Development Targets + +| Target | Description | +|---|---| +| `dev-up` | Install tools, create cluster, start Tilt | +| `dev-down` | Stop Tilt, leave cluster running | +| `dev-clean` | Stop Tilt, delete cluster and registry, remove tool binaries | +| `dev-status` | Show cluster and connection info | +| `dev-tilt-status` | Show status of all Tilt-managed resources | +| `dev-tilt-logs` | Show logs (all or `RESOURCE=`) | +| `dev-tilt-describe` | Describe a Tilt resource (`RESOURCE=`) | +| `dev-tilt-trigger` | Manually trigger a resource (`RESOURCE=`) | +| `dev-wait-ready` | Block until all resources reach Ready state | +| `dev-install-kind` | Install Kind binary to `_output/bin` | +| `dev-install-tilt` | Install Tilt binary to `_output/bin` | +| `dev-install-ctlptl` | Install ctlptl binary to `_output/bin` | +| `dev-create-kind-cluster` | Create Kind cluster and registry via ctlptl | +| `dev-delete-kind-cluster` | Delete Kind cluster and registry | + +For design rationale and architecture details, see the +[Tilt-based development design proposal](../design/tilt-based-development.md). diff --git a/docs/development/prepare-for-development.md b/docs/development/prepare-for-development.md index ae8974de411..16944b7e456 100644 --- a/docs/development/prepare-for-development.md +++ b/docs/development/prepare-for-development.md @@ -5,12 +5,46 @@ a few minutes to update this file. Volcano components only have few external dependencies you need to set up before being able to build and run the code. +- [Quick Start with Tilt (Recommended)](#quick-start-with-tilt-recommended) - [Setting up Go](#setting-up-go) - [Setting up Docker](#setting-up-docker) - [Setting up Kubernetes](#setting-up-kubernetes) - [Setting up a personal access token](#setting-up-a-personal-access-token) - [What's next?](#whats-next) + +## Quick Start with Tilt (Recommended) + +The fastest way to get a full Volcano development environment running is with the +Tilt-based workflow. It provisions a Kind cluster with a local registry and deploys +all Volcano components with live-reload — a single command gets you from zero to a +running cluster: + +```bash +make dev-up +``` + +This installs Kind, Tilt, and ctlptl into `_output/bin` (no system-wide installs), +creates a 5-node Kind cluster with a local Docker registry, deploys Volcano via Helm, +and starts Tilt for live-reloading. Edit any Go file and the running component rebuilds +automatically inside the container. + +**Prerequisites:** Docker and Make. No local Go toolchain is required for building +and running Volcano — builds happen inside containers. However, installing Go locally +is recommended for running unit tests, using code analysis tools, and working with +the project long-term. See [Setting up Go](#setting-up-go) below. + +**Note:** The first run takes approximately 20-25 minutes while Docker builds all +Go images from scratch. Subsequent runs are much faster thanks to layer caching. +See the [Tips and Best Practices](./development.md#tips-and-best-practices) section +for ways to speed up branch-switching workflows. + +For the full workflow reference (teardown, logs, tests, status), see the +[Tilt-based development section](./development.md#tilt-based-local-development) in the +development guide. For architecture and design rationale, see the +[design proposal](../design/tilt-based-development.md). + + ## Setting up Go All Volcano components are written in the [Go](https://golang.org) programming language. @@ -18,7 +52,11 @@ To build, you'll need a Go development environment. If you haven't set up a Go d environment, please follow [these instructions](https://golang.org/doc/install) to install the Go tools. -Volcano currently builds with Go 1.14 +The required go version can be tracked down from the Dockerfiles, search for "golang:" from the root: + +``` +grep -rh 'golang:' installer/dockerfile/ --include='Dockerfile*' | grep -oP 'golang:\K[0-9]+\.[0-9]+\.[0-9]+' | tail -1 +``` ## Setting up Docker From 9623ae9140a747dcf51c82e50ee64942b9c70c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Tue, 24 Feb 2026 09:06:58 +0100 Subject: [PATCH 03/21] tilt: add Kind-based local dev workflow with live-reload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Tilt-based local development environment using Kind (via ctlptl) with a local container registry. Provides single-command setup (make dev-up) with live-reload for scheduler, controller-manager, and webhook-manager components. Key features: - ctlptl-managed Kind cluster with local Docker registry - Tilt live-update with in-container Go rebuilds via entrypoint.sh - Multi-architecture support (amd64/arm64) for tool downloads, Docker builds, and in-container rebuilds - Integrated monitoring (Prometheus, Grafana) with port-forwards - On-demand unit and e2e test triggers via Tilt resources - All dev tooling self-contained in _output/bin Includes Helm chart fix: use dedicated admission_init_image_name variable for admission init Job image. Signed-off-by: Hajnal Máté --- .dockerignore | 2 + Makefile | 10 +- hack/lib/install.sh | 75 +++- hack/run-e2e-kind.sh | 92 +++-- hack/tilt/Dockerfile.admission-init.tilt | 29 ++ hack/tilt/Dockerfile.tilt | 44 +++ hack/tilt/Makefile | 157 +++++++++ hack/tilt/Tiltfile | 319 ++++++++++++++++++ hack/tilt/ctlptl-kind-registry.yaml | 10 + hack/tilt/entrypoint.sh | 47 +++ hack/tilt/values-tilt-override.yaml | 33 ++ .../volcano/templates/admission-init.yaml | 2 +- installer/helm/chart/volcano/values.yaml | 1 + 13 files changed, 776 insertions(+), 45 deletions(-) create mode 100644 .dockerignore create mode 100644 hack/tilt/Dockerfile.admission-init.tilt create mode 100644 hack/tilt/Dockerfile.tilt create mode 100644 hack/tilt/Makefile create mode 100644 hack/tilt/Tiltfile create mode 100644 hack/tilt/ctlptl-kind-registry.yaml create mode 100644 hack/tilt/entrypoint.sh create mode 100644 hack/tilt/values-tilt-override.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..301ae393f55 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +_output/ +_output diff --git a/Makefile b/Makefile index c0e2510a4f7..b88a8bb8111 100644 --- a/Makefile +++ b/Makefile @@ -64,11 +64,17 @@ endif DOCKER_PLATFORMS ?= "linux/${GOARCH}" GOOS ?= linux +KIND_VERSION ?= v0.31.0 include Makefile.def +include hack/tilt/Makefile .EXPORT_ALL_VARIABLES: +.PHONY: print-kind-version +print-kind-version: + @printf "%s\n" "${KIND_VERSION}" + all: vc-scheduler vc-agent-scheduler vc-controller-manager vc-webhook-manager vc-agent vcctl command-lines init: @@ -217,7 +223,7 @@ e2e-test-vcctl: vcctl images e2e-test-stress: images E2E_TYPE=STRESS ./hack/run-e2e-kind.sh -e2e-test-cronjob: images +e2e-test-cronjob: images E2E_TYPE=CRONJOB ./hack/run-e2e-kind.sh e2e-test-dra: images @@ -248,7 +254,7 @@ release: images generate-yaml ./hack/publish.sh clean: - rm -rf _output/ + rm -rf ${OUTPUT_DIR}/ rm -f *.log verify: diff --git a/hack/lib/install.sh b/hack/lib/install.sh index 8d25ea2abab..6d27271c355 100644 --- a/hack/lib/install.sh +++ b/hack/lib/install.sh @@ -15,11 +15,26 @@ # limitations under the License. # spin up cluster with kind command -function kind-up-cluster { +function kind-create-cluster { + local kind_cmd=${KIND_BIN:-kind} + check-kind - echo "Running kind: [kind create cluster ${CLUSTER_CONTEXT[*]} ${KIND_OPT}]" - kind create cluster "${CLUSTER_CONTEXT[@]}" ${KIND_OPT} + local cluster_name=${CLUSTER_CONTEXT[1]} + + if ${kind_cmd} get clusters 2>/dev/null | grep -qw "${cluster_name}"; then + echo "Kind cluster ${cluster_name} already exists" + else + echo "Running kind: [${kind_cmd} create cluster ${CLUSTER_CONTEXT[*]} ${KIND_OPT}]" + ${kind_cmd} create cluster "${CLUSTER_CONTEXT[@]}" ${KIND_OPT} + fi + + echo "Exporting kubeconfig for kind cluster ${cluster_name}" + ${kind_cmd} export kubeconfig --name "${cluster_name}" >/dev/null +} + +function kind-up-cluster { + kind-create-cluster echo check-images @@ -27,9 +42,9 @@ function kind-up-cluster { echo echo "Loading docker images into kind cluster" # only need to load images into control-plane node because volcano components are deployed on control-plane node. - kind load docker-image ${IMAGE_PREFIX}/vc-controller-manager:${TAG} "${CLUSTER_CONTEXT[@]}" --nodes ${CLUSTER_CONTEXT[1]}-control-plane - kind load docker-image ${IMAGE_PREFIX}/vc-scheduler:${TAG} "${CLUSTER_CONTEXT[@]}" --nodes ${CLUSTER_CONTEXT[1]}-control-plane - kind load docker-image ${IMAGE_PREFIX}/vc-webhook-manager:${TAG} "${CLUSTER_CONTEXT[@]}" --nodes ${CLUSTER_CONTEXT[1]}-control-plane + ${KIND_BIN:-kind} load docker-image ${IMAGE_PREFIX}/vc-controller-manager:${TAG} "${CLUSTER_CONTEXT[@]}" --nodes ${CLUSTER_CONTEXT[1]}-control-plane + ${KIND_BIN:-kind} load docker-image ${IMAGE_PREFIX}/vc-scheduler:${TAG} "${CLUSTER_CONTEXT[@]}" --nodes ${CLUSTER_CONTEXT[1]}-control-plane + ${KIND_BIN:-kind} load docker-image ${IMAGE_PREFIX}/vc-webhook-manager:${TAG} "${CLUSTER_CONTEXT[@]}" --nodes ${CLUSTER_CONTEXT[1]}-control-plane } # check if the required images exist @@ -66,13 +81,51 @@ function check-prerequisites { # check if kind installed function check-kind { + local kind_cmd=${KIND_BIN:-kind} + local required_version=${KIND_VERSION} + + # If KIND_VERSION is not set, try to resolve it from Makefile in VK_ROOT + if [[ -z "${required_version}" ]] && [[ -n "${VK_ROOT}" ]] && [[ -f "${VK_ROOT}/Makefile" ]]; then + required_version=$(make -s -C "${VK_ROOT}" print-kind-version 2>/dev/null) + fi + + if [[ -z "${required_version}" ]]; then + echo "ERROR: KIND_VERSION is not set and could not be resolved from Makefile" + return 1 + fi + echo "Checking kind" - which kind >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - echo "Installing kind ..." - GOOS=${OS} go install sigs.k8s.io/kind@v0.31.0 + if command -v "${kind_cmd}" >/dev/null 2>&1; then + local found_version + found_version=$("${kind_cmd}" version | grep -Eo 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1) + if [[ "${found_version}" == "${required_version}" ]]; then + echo "Found kind at ${kind_cmd}, version: ${found_version}" + return + fi + + echo "Kind version mismatch at ${kind_cmd} (found ${found_version}, required ${required_version})" + else + echo "Kind not found at ${kind_cmd}" + fi + + if [[ -n "${KIND_BIN}" ]]; then + local os arch + os=$(uname -s | tr '[:upper:]' '[:lower:]') + arch=$(uname -m) + case "${arch}" in + x86_64) arch=amd64 ;; + aarch64|arm64) arch=arm64 ;; + esac + + echo "Installing kind ${required_version} to ${KIND_BIN} ..." + mkdir -p "$(dirname "${KIND_BIN}")" + curl -fsSL "https://github.com/kubernetes-sigs/kind/releases/download/${required_version}/kind-${os}-${arch}" -o "${KIND_BIN}" + chmod +x "${KIND_BIN}" + echo "Installed kind at ${KIND_BIN}" else - echo -n "Found kind, version: " && kind version + echo "Installing kind ${required_version} via go install ..." + GOOS=${OS} go install sigs.k8s.io/kind@${required_version} + echo -n "Installed kind, version: " && kind version fi } diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh index 1a063488fdc..c92e5271c7d 100755 --- a/hack/run-e2e-kind.sh +++ b/hack/run-e2e-kind.sh @@ -24,6 +24,9 @@ export LOG_LEVEL=3 export CLEANUP_CLUSTER=${CLEANUP_CLUSTER:-1} export E2E_TYPE=${E2E_TYPE:-"ALL"} export ARTIFACTS_PATH=${ARTIFACTS_PATH:-"${VK_ROOT}/volcano-e2e-logs"} +export E2E_LOCAL_ONLY=${E2E_LOCAL_ONLY:-0} +export E2E_FOCUS=${E2E_FOCUS:-""} +export E2E_SKIP=${E2E_SKIP:-""} mkdir -p "$ARTIFACTS_PATH" NAMESPACE=${NAMESPACE:-volcano-system} @@ -241,6 +244,27 @@ function uninstall-volcano { helm uninstall "${CLUSTER_NAME}" -n ${NAMESPACE} } +function run-ginkgo-suite() { + local suite_path=$1 + local default_focus=$2 + local default_skip=$3 + shift 3 + + local focus=${E2E_FOCUS:-${default_focus}} + local skip=${E2E_SKIP:-${default_skip}} + + local cmd=(ginkgo "$@") + if [[ -n "${focus}" ]]; then + cmd+=(--focus="${focus}") + fi + if [[ -n "${skip}" ]]; then + cmd+=(--skip="${skip}") + fi + cmd+=("${suite_path}") + + KUBECONFIG=${KUBECONFIG} GOOS=${OS} "${cmd[@]}" +} + function generate-log { echo "Generating volcano log files" kind export logs "${CLUSTER_CONTEXT[@]}" "$ARTIFACTS_PATH" @@ -271,87 +295,93 @@ Disable displaying volcano component logs: exit 0 fi -if [[ $CLEANUP_CLUSTER -eq 1 ]]; then +if [[ $CLEANUP_CLUSTER -eq 1 ]] && [[ ${E2E_LOCAL_ONLY} -ne 1 ]]; then trap cleanup EXIT fi source "${VK_ROOT}/hack/lib/install.sh" -check-prerequisites -kind-up-cluster -install-kwok-with-helm +if [[ ${E2E_LOCAL_ONLY} -ne 1 ]]; then + check-prerequisites + kind-up-cluster + install-kwok-with-helm -if [[ -z ${KUBECONFIG+x} ]]; then - export KUBECONFIG="${HOME}/.kube/config" -fi + if [[ -z ${KUBECONFIG+x} ]]; then + export KUBECONFIG="${HOME}/.kube/config" + fi -install-volcano + install-volcano +else + if [[ -z ${KUBECONFIG+x} ]]; then + export KUBECONFIG="${HOME}/.kube/config" + fi +fi # Run e2e test cd ${VK_ROOT} -install-ginkgo-if-not-exist +if [[ ${E2E_LOCAL_ONLY} -ne 1 ]]; then + install-ginkgo-if-not-exist +fi case ${E2E_TYPE} in "ALL") echo "Running e2e..." - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --nodes=4 --compilers=4 --randomize-all --randomize-suites --fail-on-pending --cover --trace --race --slow-spec-threshold='30s' --progress ./test/e2e/jobp/ - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/jobseq/ - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/schedulingbase/ - # k8s 1.35 init will import its e2e suite, these k8s's suites need to skip - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --skip="\[sig-.*\]" --slow-spec-threshold='30s' --progress ./test/e2e/schedulingaction/ - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/vcctl/ - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/cronjob/ - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress --focus="DRA E2E Test" ./test/e2e/dra/ - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/admission/ - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/hypernode/ + run-ginkgo-suite ./test/e2e/jobp/ "" "" -r --nodes=4 --compilers=4 --randomize-all --randomize-suites --fail-on-pending --cover --trace --race --slow-spec-threshold='30s' --progress + run-ginkgo-suite ./test/e2e/jobseq/ "" "" -r --slow-spec-threshold='30s' --progress + run-ginkgo-suite ./test/e2e/schedulingbase/ "" "" -r --slow-spec-threshold='30s' --progress + run-ginkgo-suite ./test/e2e/schedulingaction/ "" "\[sig-.*\]" -r --slow-spec-threshold='30s' --progress + run-ginkgo-suite ./test/e2e/vcctl/ "" "" -r --slow-spec-threshold='30s' --progress + run-ginkgo-suite ./test/e2e/cronjob/ "" "" -r --slow-spec-threshold='30s' --progress + run-ginkgo-suite ./test/e2e/dra/ "DRA E2E Test" "" -r --slow-spec-threshold='30s' --progress + run-ginkgo-suite ./test/e2e/admission/ "" "" -r --slow-spec-threshold='30s' --progress + run-ginkgo-suite ./test/e2e/hypernode/ "" "" -r --slow-spec-threshold='30s' --progress ;; "JOBP") echo "Running parallel job e2e suite..." - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --nodes=4 --compilers=4 --randomize-all --randomize-suites --fail-on-pending --cover --trace --race --slow-spec-threshold='30s' --progress ./test/e2e/jobp/ + run-ginkgo-suite ./test/e2e/jobp/ "" "" -v -r --nodes=4 --compilers=4 --randomize-all --randomize-suites --fail-on-pending --cover --trace --race --slow-spec-threshold='30s' --progress ;; "JOBSEQ") echo "Running sequence job e2e suite..." - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --slow-spec-threshold='30s' --progress ./test/e2e/jobseq/ + run-ginkgo-suite ./test/e2e/jobseq/ "" "" -v -r --slow-spec-threshold='30s' --progress ;; "SCHEDULINGBASE") echo "Running scheduling base e2e suite...(need skip k8s framework's suites)" - # k8s 1.35 init will import its e2e suite, these k8s's suites need to skip - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --skip="\[sig-.*\]" --slow-spec-threshold='30s' --progress ./test/e2e/schedulingbase/ + run-ginkgo-suite ./test/e2e/schedulingbase/ "" "\[sig-.*\]" -v -r --slow-spec-threshold='30s' --progress ;; "SCHEDULINGACTION") echo "Running scheduling action e2e suite..." - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --slow-spec-threshold='30s' --progress ./test/e2e/schedulingaction/ + run-ginkgo-suite ./test/e2e/schedulingaction/ "" "" -v -r --slow-spec-threshold='30s' --progress ;; "VCCTL") echo "Running vcctl e2e suite..." - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --slow-spec-threshold='30s' --progress ./test/e2e/vcctl/ + run-ginkgo-suite ./test/e2e/vcctl/ "" "" -v -r --slow-spec-threshold='30s' --progress ;; "STRESS") echo "Running stress e2e suite..." - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --slow-spec-threshold='30s' --progress ./test/e2e/stress/ + run-ginkgo-suite ./test/e2e/stress/ "" "" -v -r --slow-spec-threshold='30s' --progress ;; "DRA") echo "Running dra e2e suite..." - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --slow-spec-threshold='30s' --progress --focus="DRA E2E Test" ./test/e2e/dra/ + run-ginkgo-suite ./test/e2e/dra/ "DRA E2E Test" "" -v -r --slow-spec-threshold='30s' --progress ;; "ADMISSION_POLICY") echo "Running admission policy e2e suite..." - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --slow-spec-threshold='30s' --progress ./test/e2e/admission/ + run-ginkgo-suite ./test/e2e/admission/ "" "" -v -r --slow-spec-threshold='30s' --progress ;; "ADMISSION_WEBHOOK") echo "Running admission webhook e2e suite..." - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --slow-spec-threshold='30s' --progress ./test/e2e/admission/ + run-ginkgo-suite ./test/e2e/admission/ "" "" -v -r --slow-spec-threshold='30s' --progress ;; "HYPERNODE") echo "Creating 8 kwok nodes for 3-tier topology" install-kwok-nodes 8 echo "Running hypernode e2e suite..." - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/hypernode/ + run-ginkgo-suite ./test/e2e/hypernode/ "" "" -r --slow-spec-threshold='30s' --progress ;; "CRONJOB") echo "Running cronjob e2e suite..." - KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --slow-spec-threshold='30s' --progress ./test/e2e/cronjob/ + run-ginkgo-suite ./test/e2e/cronjob/ "" "" -v -r --slow-spec-threshold='30s' --progress ;; esac diff --git a/hack/tilt/Dockerfile.admission-init.tilt b/hack/tilt/Dockerfile.admission-init.tilt new file mode 100644 index 00000000000..27e78ccb56e --- /dev/null +++ b/hack/tilt/Dockerfile.admission-init.tilt @@ -0,0 +1,29 @@ +# Copyright 2025 The Volcano Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM alpine:latest +ARG KUBE_VERSION="1.35.0" +ARG TARGETARCH +ARG APK_MIRROR +RUN if [ -n "$APK_MIRROR" ]; then sed -i "s@https://dl-cdn.alpinelinux.org@${APK_MIRROR}@g" /etc/apk/repositories ; fi && \ + apk add --update ca-certificates && \ + apk add --update openssl && \ + apk add --update -t deps curl && \ + curl -L https://dl.k8s.io/release/v$KUBE_VERSION/bin/linux/$TARGETARCH/kubectl -o /usr/local/bin/kubectl && \ + chmod +x /usr/local/bin/kubectl && \ + apk del --purge deps && \ + rm /var/cache/apk/* + +ADD installer/dockerfile/webhook-manager/gen-admission-secret.sh /gen-admission-secret.sh +ENTRYPOINT ["/gen-admission-secret.sh"] diff --git a/hack/tilt/Dockerfile.tilt b/hack/tilt/Dockerfile.tilt new file mode 100644 index 00000000000..9340e5f7265 --- /dev/null +++ b/hack/tilt/Dockerfile.tilt @@ -0,0 +1,44 @@ +# Copyright 2025 The Volcano Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.25.0 + +ARG TARGETARCH +ARG COMPONENT +ENV COMPONENT=${COMPONENT} +ENV VOLCANO_HOME=/go/src/volcano.sh/volcano + +RUN apt-get update && \ + apt-get install -y ca-certificates openssl curl dumb-init less && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR ${VOLCANO_HOME} +COPY go.mod go.sum ./ +# Copy staging directory before go mod download since go.mod has replace directive +COPY staging/ ./staging/ +RUN go mod download + +COPY pkg/ ./pkg/ +COPY third_party/ ./third_party/ +COPY cmd/ ./cmd/ +COPY hack/tilt/entrypoint.sh ${VOLCANO_HOME}/entrypoint.sh + +RUN chmod +x ${VOLCANO_HOME}/entrypoint.sh +RUN chown -R 1000:1000 ${VOLCANO_HOME} +RUN useradd -u 1000 -m developer +USER developer + +RUN GOOS=linux GOARCH=${TARGETARCH} go build -o ${VOLCANO_HOME}/vc-${COMPONENT} ./cmd/${COMPONENT} + +ENTRYPOINT ["/usr/bin/dumb-init", "-c", "--", "/go/src/volcano.sh/volcano/entrypoint.sh"] diff --git a/hack/tilt/Makefile b/hack/tilt/Makefile new file mode 100644 index 00000000000..08352c15c62 --- /dev/null +++ b/hack/tilt/Makefile @@ -0,0 +1,157 @@ +# Copyright 2025 The Volcano Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This Makefile expects the following variables to be defined by the parent/root Makefile: +# OUTPUT_DIR, BIN_DIR +ifndef OUTPUT_DIR +$(error OUTPUT_DIR is not set. Please define it in the parent Makefile.) +endif + +ifndef BIN_DIR +$(error BIN_DIR is not set. Please define it in the parent Makefile.) +endif + + +# Map host CPU to release archive naming: aarch64 becomes arm64 to match +# upstream tarball names. x86_64 is intentionally kept as-is because Tilt +# and ctlptl archives use "linux.x86_64" (not "amd64") in their filenames. +HOSTARCH := $(shell uname -m) +ifeq ($(HOSTARCH),aarch64) +HOSTARCH := arm64 +endif + +TILT_VERSION=0.36.3 +CTLPTL_VERSION ?= 0.9.0 +TILT_BIN=${BIN_DIR}/tilt +KIND_BIN=${BIN_DIR}/kind +CTLPTL_BIN=${BIN_DIR}/ctlptl + +ifndef KIND_VERSION +$(error KIND_VERSION is not set. Please define it in the parent Makefile.) +endif + +KIND_CLUSTER_NAME ?= kind-volcano-dev +KIND_CONTEXT_NAME ?= ${KIND_CLUSTER_NAME} +CTLPTL_CONFIG ?= hack/tilt/ctlptl-kind-registry.yaml +KIND_CONFIG ?= hack/e2e-kind-config.yaml + +TILT_WAIT_TIMEOUT ?= 10m +TILT_WAIT_RESOURCES ?= \ + uiresource/volcano-namespaces \ + uiresource/volcano-crds \ + uiresource/volcano-vcjob-roles \ + uiresource/volcano-admission-init \ + uiresource/volcano-admission \ + uiresource/volcano-scheduler \ + uiresource/volcano-controllers \ + uiresource/prometheus-deployment \ + uiresource/kube-state-metrics \ + uiresource/grafana + +# Install Kind if not present in _output/bin or wrong version +dev-install-kind: + @PATH="${BIN_DIR}:$$PATH" KIND_BIN="${KIND_BIN}" KIND_VERSION="${KIND_VERSION}" bash -c 'source hack/lib/install.sh; check-kind' + +# Install ctlptl if not present in _output/bin or wrong version +dev-install-ctlptl: + @if [ ! -f ${CTLPTL_BIN} ] || [ "$$(${CTLPTL_BIN} version 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+' | head -1)" != "${CTLPTL_VERSION}" ]; then \ + mkdir -p ${BIN_DIR}; \ + curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v${CTLPTL_VERSION}/ctlptl.${CTLPTL_VERSION}.linux.${HOSTARCH}.tar.gz | tar -xz -C ${BIN_DIR} ctlptl; \ + chmod +x ${CTLPTL_BIN}; \ + printf "ctlptl installed at ${CTLPTL_BIN} (version ${CTLPTL_VERSION}).\n"; \ + else \ + printf "ctlptl already installed at ${CTLPTL_BIN} (version ${CTLPTL_VERSION}).\n"; \ + fi + +# Create Kind cluster and registry for local dev via ctlptl. +# Composes the ctlptl config with the shared kind cluster config at apply time +# so that e2e-kind-config.yaml remains the single source of truth. +dev-create-kind-cluster: + @{ \ + cat ${CTLPTL_CONFIG}; \ + printf 'kindV1Alpha4Cluster:\n'; \ + printf ' name: %s\n' '${KIND_CLUSTER_NAME}'; \ + sed -e '/^kind:/d' -e '/^apiVersion:/d' -e '/^[[:space:]]*$$/d' -e '/^#/d' -e 's/^/ /' ${KIND_CONFIG}; \ + } | ${CTLPTL_BIN} apply -f - + +.PHONY: dev-up +dev-up: dev-install-tilt dev-install-kind dev-install-ctlptl + $(MAKE) dev-create-kind-cluster + $(TILT_BIN) up -f hack/tilt/Tiltfile + +.PHONY: dev-down +dev-down: + @$(TILT_BIN) down -f hack/tilt/Tiltfile 2>/dev/null || true + @pkill -f '^${TILT_BIN} up -f hack/tilt/Tiltfile$$' || true + +# Delete Kind cluster and registry managed by ctlptl +.PHONY: dev-delete-kind-cluster +dev-delete-kind-cluster: + @${CTLPTL_BIN} delete -f ${CTLPTL_CONFIG} --cascade=true --ignore-not-found || true + +.PHONY: dev-clean +dev-clean: dev-down dev-delete-kind-cluster + rm -f $(KIND_BIN) $(CTLPTL_BIN) $(TILT_BIN) + +.PHONY: dev-status +dev-status: + ${CTLPTL_BIN} get cluster + @echo --- + @kubectl cluster-info --context ${KIND_CONTEXT_NAME} 2>/dev/null || kubectl cluster-info --context kind-${KIND_CLUSTER_NAME} 2>/dev/null || echo "Cluster not running." + +# Install Tilt if not present in _output/bin or wrong version +dev-install-tilt: + @if [ ! -f ${TILT_BIN} ] || [ "$$(${TILT_BIN} version | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+' | head -1)" != "${TILT_VERSION}" ]; then \ + mkdir -p ${BIN_DIR}; \ + TMP_DIR=$$(mktemp -d); \ + curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/tilt.${TILT_VERSION}.linux.${HOSTARCH}.tar.gz | tar -xz -C $$TMP_DIR && \ + mv $$TMP_DIR/tilt ${TILT_BIN}; \ + rm -rf $$TMP_DIR; \ + chmod +x ${TILT_BIN}; \ + printf "Tilt installed at ${TILT_BIN} (version ${TILT_VERSION}).\n"; \ + else \ + printf "Tilt already installed at ${TILT_BIN} (version ${TILT_VERSION}).\n"; \ + fi + +.PHONY: dev-tilt-status +dev-tilt-status: dev-install-tilt + ${TILT_BIN} get uiresource + +.PHONY: dev-tilt-logs +dev-tilt-logs: dev-install-tilt + @if [ -n "${RESOURCE}" ]; then \ + ${TILT_BIN} logs ${RESOURCE}; \ + else \ + ${TILT_BIN} logs; \ + fi + +.PHONY: dev-tilt-describe +dev-tilt-describe: dev-install-tilt + @if [ -z "${RESOURCE}" ]; then \ + echo "RESOURCE is required. Example: make dev-tilt-describe RESOURCE=volcano-scheduler"; \ + exit 1; \ + fi + ${TILT_BIN} describe uiresource ${RESOURCE} + +.PHONY: dev-tilt-trigger +dev-tilt-trigger: dev-install-tilt + @if [ -z "${RESOURCE}" ]; then \ + echo "RESOURCE is required. Example: make dev-tilt-trigger RESOURCE=volcano-scheduler"; \ + exit 1; \ + fi + ${TILT_BIN} trigger ${RESOURCE} + +.PHONY: dev-wait-ready +dev-wait-ready: dev-install-tilt + ${TILT_BIN} wait --for=condition=Ready ${TILT_WAIT_RESOURCES} --timeout=${TILT_WAIT_TIMEOUT} diff --git a/hack/tilt/Tiltfile b/hack/tilt/Tiltfile new file mode 100644 index 00000000000..9e87da7ec0a --- /dev/null +++ b/hack/tilt/Tiltfile @@ -0,0 +1,319 @@ +# Tiltfile for Volcano + +# Copyright 2025 The Volcano Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ensure Kind cluster exists for local dev +kind_cluster_name = 'kind-volcano-dev' +kind_k8s_context = kind_cluster_name +allow_k8s_contexts([kind_k8s_context]) +e2e_local_cluster_env = 'CLUSTER_NAME=${CLUSTER_NAME:-' + kind_cluster_name + '} ' + +# Set the default namespace for all resources +k8s_namespace('volcano-system') + +# Apply namespace manifest +root_dir = '../../' +k8s_yaml(root_dir + 'installer/namespace.yaml') + +# Helm-based installation (using helm() pattern) +volcano_yaml = helm( + root_dir + 'installer/helm/chart/volcano', + name='volcano', + namespace='volcano-system', + values=[root_dir + 'installer/helm/chart/volcano/values.yaml', 'values-tilt-override.yaml'], + set=[ + 'basic.admission_init_image_name=volcanosh/vc-admission-init', + ] +) +k8s_yaml(volcano_yaml) + +k8s_yaml(root_dir + 'installer/volcano-monitoring.yaml') + +# Build all images for local registry +images = [ + ('volcanosh/vc-scheduler', root_dir, 'scheduler'), + ('volcanosh/vc-controller-manager', root_dir, 'controller-manager'), + ('volcanosh/vc-webhook-manager', root_dir, 'webhook-manager'), +] +for name, path, component in images: + docker_build( + name, + path, + build_args=dict(COMPONENT=component), + dockerfile='Dockerfile.tilt', + only=['pkg/', 'third_party/', 'cmd/'+ component + '/', 'staging/', 'hack/tilt/entrypoint.sh', 'go.mod', 'go.sum'], + live_update=[ + sync(root_dir + 'pkg', '/go/src/volcano.sh/volcano/pkg'), + sync(root_dir + 'third_party', '/go/src/volcano.sh/volcano/third_party'), + sync(root_dir + 'cmd/' + component, '/go/src/volcano.sh/volcano/cmd/' + component), + sync(root_dir + 'hack/tilt/entrypoint.sh', '/go/src/volcano.sh/volcano/entrypoint.sh'), + run('touch /tmp/reload') + ] + ) + +docker_build( + 'volcanosh/vc-admission-init', + root_dir, + dockerfile='Dockerfile.admission-init.tilt', + only=["installer/dockerfile/webhook-manager/gen-admission-secret.sh"], +) + + +# Group all namespaces +k8s_resource( + new_name='volcano-namespaces', + objects=[ + 'volcano-system:namespace', + 'volcano-monitoring:namespace', + ], + labels=["volcano"] +) + +k8s_resource( + new_name='volcano-crds', + objects=[ + 'jobtemplates.flow.volcano.sh:customresourcedefinition', + 'jobflows.flow.volcano.sh:customresourcedefinition', + 'jobs.batch.volcano.sh:customresourcedefinition', + 'commands.bus.volcano.sh:customresourcedefinition', + 'numatopologies.nodeinfo.volcano.sh:customresourcedefinition', + 'podgroups.scheduling.volcano.sh:customresourcedefinition', + 'queues.scheduling.volcano.sh:customresourcedefinition', + 'hypernodes.topology.volcano.sh:customresourcedefinition', + 'cronjobs.batch.volcano.sh:customresourcedefinition', + 'nodeshards.shard.volcano.sh:customresourcedefinition', + 'colocationconfigurations.config.volcano.sh:customresourcedefinition' + ], + labels=["volcano"] +) + + +# Group all admission-related resources +k8s_resource( + 'volcano-admission', + objects=[ + 'volcano-admission:serviceaccount', + 'volcano-admission:clusterrole', + 'volcano-admission-role:clusterrolebinding', + 'volcano-admission-configmap:configmap', + 'volcano-admission-service-queues-mutate:mutatingwebhookconfiguration', + 'volcano-admission-service-jobs-mutate:mutatingwebhookconfiguration', + 'volcano-admission-service-jobs-validate:validatingwebhookconfiguration', + 'volcano-admission-service-queues-validate:validatingwebhookconfiguration', + 'volcano-admission-service-podgroups-validate:validatingwebhookconfiguration', + 'volcano-admission-service-hypernodes-validate:validatingwebhookconfiguration', + 'volcano-admission-service-cronjobs-validate:validatingwebhookconfiguration' + ], + resource_deps=['volcano-namespaces', 'volcano-crds', 'volcano-admission-init'], + labels=["volcano"] +) + +# Group scheduler resources +k8s_resource( + 'volcano-scheduler', + objects=[ + 'volcano-scheduler:serviceaccount', + 'volcano-scheduler:clusterrole', + 'volcano-scheduler-role:clusterrolebinding', + 'volcano-scheduler-configmap:configmap', + ], + resource_deps=['volcano-namespaces'], + labels=["volcano"] +) + +# Group controller resources +k8s_resource( + 'volcano-controllers', + objects=[ + 'volcano-controllers:serviceaccount', + 'volcano-controllers:clusterrole', + 'volcano-controllers-role:clusterrolebinding', + 'volcano-controller-configmap:configmap' + ], + resource_deps=['volcano-namespaces'], + labels=["volcano"] +) + +k8s_resource( + new_name='volcano-vcjob-roles', + objects=[ + 'vcjob-editor-role:clusterrole', + 'vcjob-viewer-role:clusterrole' + ], + labels=["volcano"] +) + +k8s_resource( + 'volcano-admission-init', + extra_pod_selectors=[{'job-name': 'volcano-admission-init'}], + objects=[ + 'volcano-admission-init:serviceaccount', + 'volcano-admission-init:role', + 'volcano-admission-init-role:rolebinding', + ], + resource_deps=['volcano-namespaces'], + labels=["volcano"] +) + +k8s_resource( + 'prometheus-deployment', + objects=[ + 'prometheus-volcano:clusterrolebinding', + 'prometheus-volcano:clusterrole', + 'prometheus-server-conf:configmap' + ], + resource_deps=['volcano-namespaces'], + port_forwards=3001, + labels=["monitoring"] +) + +k8s_resource( + 'kube-state-metrics', + objects=[ + 'kube-state-metrics:serviceaccount', + 'kube-state-metrics:clusterrole', + 'kube-state-metrics:clusterrolebinding', + ], + resource_deps=['volcano-namespaces'], + labels=["monitoring"] +) + +k8s_resource( + 'grafana', + objects=[ + 'grafana-datasources:configmap', + 'grafana-volcano-dashboard-config:configmap', + 'grafana-volcano-dashboard:configmap', + ], + resource_deps=['volcano-namespaces'], + port_forwards=3000, + labels=["monitoring"] +) + +local_resource( + 'install-kwok', + 'bash -c "source ' + root_dir + 'hack/lib/install.sh && install-kwok-with-helm"', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests-install"] +) + +local_resource( + 'install-ginkgo', + 'bash -c "source ' + root_dir + 'hack/lib/install.sh && install-ginkgo-if-not-exist"', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests-install"] +) + +# Optionally, run e2e tests (using local_resource) +local_resource( + 'unit-tests', + 'make unit-test', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests"] +) + +local_resource( + 'e2e-tests-all', + e2e_local_cluster_env + 'E2E_LOCAL_ONLY=1 E2E_TYPE=ALL KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} ' + root_dir + 'hack/run-e2e-kind.sh', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests"], + resource_deps=['install-kwok', 'install-ginkgo'] +) + +local_resource( + 'e2e-tests-jobp', + e2e_local_cluster_env + 'E2E_LOCAL_ONLY=1 E2E_TYPE=JOBP KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} ' + root_dir + 'hack/run-e2e-kind.sh', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests"], + resource_deps=['install-ginkgo'] +) + +local_resource( + 'e2e-tests-jobseq', + e2e_local_cluster_env + 'E2E_LOCAL_ONLY=1 E2E_TYPE=JOBSEQ KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} ' + root_dir + 'hack/run-e2e-kind.sh', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests"], + resource_deps=['install-ginkgo'] +) + +local_resource( + 'e2e-tests-schedulingbase', + e2e_local_cluster_env + 'E2E_LOCAL_ONLY=1 E2E_TYPE=SCHEDULINGBASE KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} ' + root_dir + 'hack/run-e2e-kind.sh', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests"], + resource_deps=['install-ginkgo'] +) + +local_resource( + 'e2e-tests-schedulingaction', + e2e_local_cluster_env + 'E2E_LOCAL_ONLY=1 E2E_TYPE=SCHEDULINGACTION KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} ' + root_dir + 'hack/run-e2e-kind.sh', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests"], + resource_deps=['install-ginkgo'] +) + +local_resource( + 'e2e-tests-vcctl', + e2e_local_cluster_env + 'E2E_LOCAL_ONLY=1 E2E_TYPE=VCCTL KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} ' + root_dir + 'hack/run-e2e-kind.sh', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests"], + resource_deps=['install-ginkgo'] +) + +local_resource( + 'e2e-tests-dra', + e2e_local_cluster_env + 'E2E_LOCAL_ONLY=1 E2E_TYPE=DRA KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} ' + root_dir + 'hack/run-e2e-kind.sh', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests"], + resource_deps=['install-ginkgo'] +) + +local_resource( + 'e2e-tests-hypernode', + e2e_local_cluster_env + 'E2E_LOCAL_ONLY=1 E2E_TYPE=HYPERNODE KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} ' + root_dir + 'hack/run-e2e-kind.sh', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests"], + resource_deps=['install-ginkgo'] +) + +local_resource( + 'e2e-tests-cronjob', + e2e_local_cluster_env + 'E2E_LOCAL_ONLY=1 E2E_TYPE=CRONJOB KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} ' + root_dir + 'hack/run-e2e-kind.sh', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests"], + resource_deps=['install-ginkgo'] +) + +# Focused example (single-test run) +local_resource( + 'e2e-tests-schedulingbase-focused-example', + e2e_local_cluster_env + 'E2E_LOCAL_ONLY=1 E2E_TYPE=SCHEDULINGBASE E2E_FOCUS="${E2E_FOCUS:-Gang scheduling}" E2E_SKIP="${E2E_SKIP:-\\[sig-.*\\]|Full Occupied}" KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} ' + root_dir + 'hack/run-e2e-kind.sh', + trigger_mode=TRIGGER_MODE_MANUAL, + auto_init=False, + labels=["tests"], + resource_deps=['install-ginkgo'] +) diff --git a/hack/tilt/ctlptl-kind-registry.yaml b/hack/tilt/ctlptl-kind-registry.yaml new file mode 100644 index 00000000000..1de4c1d259c --- /dev/null +++ b/hack/tilt/ctlptl-kind-registry.yaml @@ -0,0 +1,10 @@ +apiVersion: ctlptl.dev/v1alpha1 +kind: Registry +name: volcano-registry +port: 5005 +--- +apiVersion: ctlptl.dev/v1alpha1 +kind: Cluster +name: kind-volcano-dev +product: kind +registry: volcano-registry diff --git a/hack/tilt/entrypoint.sh b/hack/tilt/entrypoint.sh new file mode 100644 index 00000000000..9172e0df99d --- /dev/null +++ b/hack/tilt/entrypoint.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2025 The Volcano Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SRC_DIR=/go/src/volcano.sh/volcano +BIN="${SRC_DIR}/vc-${COMPONENT}" +RELOAD_FILE="/tmp/reload" + +child_pid=0 + +start_child() { + "$BIN" "$@" & + child_pid=$! +} + +stop_child() { + if [ $child_pid -ne 0 ]; then + kill $child_pid + wait $child_pid 2>/dev/null || true + fi +} + +start_child "$@" + +while true; do + if [ -f "$RELOAD_FILE" ]; then + echo "[entrypoint] Detected reload trigger, rebuilding and restarting..." + stop_child + GOOS=linux GOARCH="$(go env GOARCH)" go build -o "$BIN" ./cmd/${COMPONENT} + start_child "$@" + rm -f "$RELOAD_FILE" + fi + sleep 1 +done diff --git a/hack/tilt/values-tilt-override.yaml b/hack/tilt/values-tilt-override.yaml new file mode 100644 index 00000000000..4a70a620338 --- /dev/null +++ b/hack/tilt/values-tilt-override.yaml @@ -0,0 +1,33 @@ +basic: + image_pull_policy: IfNotPresent + scheduler_config_file: config/volcano-scheduler-ci.conf + +custom: + controller_log_level: 5 + scheduler_log_level: 5 + scheduler_schedule_period: 1s + admission_tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + controller_tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + scheduler_tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + default_ns: + node-role.kubernetes.io/control-plane: "" + # scheduler_feature_gates: ${FEATURE_GATES} + # ignored_provisioners: ${IGNORED_PROVISIONERS:-""} diff --git a/installer/helm/chart/volcano/templates/admission-init.yaml b/installer/helm/chart/volcano/templates/admission-init.yaml index 3a7c94f5032..9a97b3f9440 100644 --- a/installer/helm/chart/volcano/templates/admission-init.yaml +++ b/installer/helm/chart/volcano/templates/admission-init.yaml @@ -107,7 +107,7 @@ spec: resources: {{- toYaml .Values.custom.admission_resources | nindent 12 }} {{- end }} - image: {{ .Values.basic.image_registry }}/{{.Values.basic.admission_image_name}}:{{.Values.basic.image_tag_version}} + image: {{ .Values.basic.image_registry }}/{{.Values.basic.admission_init_image_name}}:{{.Values.basic.image_tag_version}} imagePullPolicy: {{ .Values.basic.image_pull_policy }} command: ["./gen-admission-secret.sh", "--service", "{{ .Release.Name }}-admission-service", "--namespace", "{{ .Release.Namespace }}", "--secret", "{{.Values.basic.admission_secret_name}}"] diff --git a/installer/helm/chart/volcano/values.yaml b/installer/helm/chart/volcano/values.yaml index c29781fe8f7..29d1e4158e9 100644 --- a/installer/helm/chart/volcano/values.yaml +++ b/installer/helm/chart/volcano/values.yaml @@ -3,6 +3,7 @@ basic: scheduler_image_name: "volcanosh/vc-scheduler" agent_scheduler_image_name: "volcanosh/vc-agent-scheduler" admission_image_name: "volcanosh/vc-webhook-manager" + admission_init_image_name: "volcanosh/vc-webhook-manager" agent_image_name: "volcanosh/vc-agent" admission_secret_name: "volcano-admission-secret" admission_config_file: "config/volcano-admission.conf" From 0065bfbeb91d0af4220910526dbb361aee1d9d71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Thu, 25 Sep 2025 12:18:19 +0200 Subject: [PATCH 04/21] fix: cache util fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The checking functions in the cache utils had ambiguous logging entries: I0925 07:48:34.403937 5756 util.go:79] schedulerPodName is responsible to Node k3d-volcano-dev-agent-2 Since in single scheduler scenarios the schedulerPodname is not populated. These entries are intimidating, because: - It's not clear that we are in caching territory so no things to be afraid. - Looks weird that there is an empty string after schedulerPodName. Furthermore, if it's empty the consistent hash circle is not needed to be checked. Signed-off-by: Hajnal Máté --- pkg/scheduler/cache/util.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pkg/scheduler/cache/util.go b/pkg/scheduler/cache/util.go index f6aae151477..80aa00f9d96 100644 --- a/pkg/scheduler/cache/util.go +++ b/pkg/scheduler/cache/util.go @@ -37,6 +37,15 @@ const ( hyperNodeEventSourceHyperNode hyperNodeEventSource = "hyperNode" ) +// isSingleSchedulerScenario checks if it's a single scheduler scenario and logs it. +func isSingleSchedulerScenario(mySchedulerPodName, objectType, objectName string) bool { + if mySchedulerPodName == "" { + klog.V(5).Infof("No schedulerPodName specified for %s %s (not a multi-scheduler scenario)", objectType, objectName) + return true + } + return false +} + // responsibleForPod returns false at following conditions: // 1. The current scheduler is not specified scheduler in Pod's spec. // 2. The Job which the Pod belongs is not assigned to current scheduler based on the hash algorithm in multi-schedulers scenario @@ -44,6 +53,11 @@ func responsibleForPod(pod *v1.Pod, schedulerNames []string, mySchedulerPodName if !slices.Contains(schedulerNames, pod.Spec.SchedulerName) { return false } + + if isSingleSchedulerScenario(mySchedulerPodName, "pod", fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)) { + return true + } + if c != nil { var key string if len(pod.OwnerReferences) != 0 { @@ -66,6 +80,10 @@ func responsibleForPod(pod *v1.Pod, schedulerNames []string, mySchedulerPodName // responsibleForNode returns true if the Node is assigned to current scheduler in multi-scheduler scenario func responsibleForNode(nodeName string, mySchedulerPodName string, c *consistent.Consistent) bool { + if isSingleSchedulerScenario(mySchedulerPodName, "node", nodeName) { + return true + } + if c != nil { schedulerPodName, err := c.Get(nodeName) if err != nil { @@ -82,6 +100,10 @@ func responsibleForNode(nodeName string, mySchedulerPodName string, c *consisten // responsibleForPodGroup returns true if Job which PodGroup belongs is assigned to current scheduler in multi-schedulers scenario func responsibleForPodGroup(pg *scheduling.PodGroup, mySchedulerPodName string, c *consistent.Consistent) bool { + if isSingleSchedulerScenario(mySchedulerPodName, "podGroup", fmt.Sprintf("%s/%s", pg.Namespace, pg.Name)) { + return true + } + if c != nil { var key string if len(pg.OwnerReferences) != 0 { From e22b49dbf7fd671371c31090e619fecf5c5beb0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Mon, 16 Mar 2026 15:33:58 +0100 Subject: [PATCH 05/21] enhancement(victim selection): add job ordering tie-break with task order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply plugin job CompareFn first in BuildVictimsPriorityQueue and use TaskOrderFn as the tie-breaker when job ordering is equal, including preemptor-missing paths, so creation timestamp does not dominate same-priority victim selection. Signed-off-by: Hajnal Máté --- pkg/scheduler/framework/session_plugins.go | 25 +++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/pkg/scheduler/framework/session_plugins.go b/pkg/scheduler/framework/session_plugins.go index e17e215795c..6fa126a8f64 100644 --- a/pkg/scheduler/framework/session_plugins.go +++ b/pkg/scheduler/framework/session_plugins.go @@ -656,8 +656,7 @@ func (ssn *Session) SubJobOrderFn(l, r interface{}) bool { return lv.UID < rv.UID } -// JobOrderFn invoke joborder function of the plugins -func (ssn *Session) JobOrderFn(l, r interface{}) bool { +func (ssn *Session) JobOrderCompareFn(l, r interface{}) int { for _, tier := range ssn.Tiers { for _, plugin := range tier.Plugins { if !isEnabled(plugin.EnabledJobOrder) { @@ -668,11 +667,20 @@ func (ssn *Session) JobOrderFn(l, r interface{}) bool { continue } if j := jof(l, r); j != 0 { - return j < 0 + return j } } } + return 0 +} + +// JobOrderFn invoke joborder function of the plugins +func (ssn *Session) JobOrderFn(l, r interface{}) bool { + if res := ssn.JobOrderCompareFn(l, r); res != 0 { + return res < 0 + } + // If no job order funcs, order job by CreationTimestamp first, then by UID. lv := l.(*api.JobInfo) rv := r.(*api.JobInfo) @@ -1090,6 +1098,13 @@ func (ssn *Session) HyperNodeGradientForSubJobFn(subJob *api.SubJobInfo, hyperNo // if victims has same job id, sorted by !ssn.TaskOrderFn // if victims has different job id, sorted by !ssn.JobOrderFn func (ssn *Session) BuildVictimsPriorityQueue(victims []*api.TaskInfo, preemptor *api.TaskInfo) *util.PriorityQueue { + jobThenTaskOrder := func(lvJob, rvJob *api.JobInfo, l, r interface{}) bool { + if cmp := ssn.JobOrderCompareFn(lvJob, rvJob); cmp != 0 { + return cmp > 0 + } + return !ssn.TaskOrderFn(l, r) + } + victimsQueue := util.NewPriorityQueue(func(l, r interface{}) bool { lv := l.(*api.TaskInfo) rv := r.(*api.TaskInfo) @@ -1121,14 +1136,14 @@ func (ssn *Session) BuildVictimsPriorityQueue(victims []*api.TaskInfo, preemptor } if !preemptorJobFound { - return !ssn.JobOrderFn(lvJob, rvJob) + return jobThenTaskOrder(lvJob, rvJob, l, r) } if lvJob.Queue != rvJob.Queue { return ssn.VictimQueueOrderFn(ssn.Queues[lvJob.Queue], ssn.Queues[rvJob.Queue], ssn.Queues[preemptorJob.Queue]) } - return !ssn.JobOrderFn(lvJob, rvJob) + return jobThenTaskOrder(lvJob, rvJob, l, r) }) for _, victim := range victims { victimsQueue.Push(victim) From 96507affebd17114108d3164a6e74995b4b481f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Mon, 16 Mar 2026 16:08:22 +0100 Subject: [PATCH 06/21] test(victim selection): add tie-break coverage in framework_test package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add BuildVictimsPriorityQueue unit coverage for job-order ties falling back to task-order, including preemptor-found and preemptor-missing paths. Place the test in package framework_test so it can use real plugin wiring without creating an import cycle between framework and plugin packages. Signed-off-by: Hajnal Máté --- .../session_plugins_victim_order_test.go | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 pkg/scheduler/framework/session_plugins_victim_order_test.go diff --git a/pkg/scheduler/framework/session_plugins_victim_order_test.go b/pkg/scheduler/framework/session_plugins_victim_order_test.go new file mode 100644 index 00000000000..dbffe605035 --- /dev/null +++ b/pkg/scheduler/framework/session_plugins_victim_order_test.go @@ -0,0 +1,117 @@ +/* +Copyright 2026 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + + schedulingv1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" + "volcano.sh/volcano/pkg/scheduler/api" + "volcano.sh/volcano/pkg/scheduler/conf" + "volcano.sh/volcano/pkg/scheduler/framework" + "volcano.sh/volcano/pkg/scheduler/plugins/priority" + "volcano.sh/volcano/pkg/scheduler/uthelper" + "volcano.sh/volcano/pkg/scheduler/util" +) + +func TestBuildVictimsPriorityQueueJobTieBreaksWithTaskOrder(t *testing.T) { + trueValue := true + plugins := map[string]framework.PluginBuilder{priority.PluginName: priority.New} + highPri, lowPri := int32(100), int32(1) + queueQ1 := util.BuildQueue("q1", 1, nil) + nodeN1 := util.BuildNode("n1", api.BuildResourceList("10", "10Gi", []api.ScalarResource{{Name: "pods", Value: "20"}}...), nil) + pgHigh := util.BuildPodGroup("pg-high", "ns1", "q1", 1, nil, schedulingv1.PodGroupRunning) + pgLow := util.BuildPodGroup("pg-low", "ns1", "q1", 1, nil, schedulingv1.PodGroupRunning) + pgPreemptor := util.BuildPodGroup("pg-preemptor", "ns1", "q1", 1, nil, schedulingv1.PodGroupPending) + pHigh := util.BuildPodWithPriority("ns1", "p-high", "n1", v1.PodRunning, api.BuildResourceList("1", "1Gi"), "pg-high", nil, nil, &highPri) + pLow := util.BuildPodWithPriority("ns1", "p-low", "n1", v1.PodRunning, api.BuildResourceList("1", "1Gi"), "pg-low", nil, nil, &lowPri) + pPreemptor := util.BuildPodWithPriority("ns1", "p-preemptor", "", v1.PodPending, api.BuildResourceList("1", "1Gi"), "pg-preemptor", nil, nil, &highPri) + + tiers := []conf.Tier{{ + Plugins: []conf.PluginOption{{ + Name: priority.PluginName, + EnabledJobOrder: &trueValue, + EnabledTaskOrder: &trueValue, + }}, + }} + + findTask := func(ssn *framework.Session, podName string) *api.TaskInfo { + for _, job := range ssn.Jobs { + for _, task := range job.Tasks { + if task.Name == podName { + return task + } + } + } + return nil + } + + baseTestStruct := func() uthelper.TestCommonStruct { + return uthelper.TestCommonStruct{ + Plugins: plugins, + Queues: []*schedulingv1.Queue{queueQ1.DeepCopy()}, + Nodes: []*v1.Node{nodeN1.DeepCopy()}, + PodGroups: []*schedulingv1.PodGroup{ + pgHigh.DeepCopy(), + pgLow.DeepCopy(), + }, + Pods: []*v1.Pod{ + pHigh.DeepCopy(), + pLow.DeepCopy(), + }, + } + } + + t.Run("preemptor job found", func(t *testing.T) { + tc := baseTestStruct() + tc.PodGroups = append(tc.PodGroups, pgPreemptor.DeepCopy()) + tc.Pods = append(tc.Pods, pPreemptor.DeepCopy()) + ssn := tc.RegisterSession(tiers, nil) + defer tc.Close() + + high := findTask(ssn, "p-high") + low := findTask(ssn, "p-low") + preemptor := findTask(ssn, "p-preemptor") + assert.NotNil(t, high) + assert.NotNil(t, low) + assert.NotNil(t, preemptor) + + victims := []*api.TaskInfo{high, low} + pq := ssn.BuildVictimsPriorityQueue(victims, preemptor) + first := pq.Pop().(*api.TaskInfo) + assert.Equal(t, "p-low", first.Name) + }) + + t.Run("preemptor job missing", func(t *testing.T) { + tc := baseTestStruct() + ssn := tc.RegisterSession(tiers, nil) + defer tc.Close() + + high := findTask(ssn, "p-high") + low := findTask(ssn, "p-low") + assert.NotNil(t, high) + assert.NotNil(t, low) + + victims := []*api.TaskInfo{high, low} + pq := ssn.BuildVictimsPriorityQueue(victims, &api.TaskInfo{Job: api.JobID("missing")}) + first := pq.Pop().(*api.TaskInfo) + assert.Equal(t, "p-low", first.Name) + }) +} From a2df7db27d700eca1310c102c05470613b72b3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Mon, 16 Mar 2026 22:54:36 +0100 Subject: [PATCH 07/21] test(victim selection): strengthen tie-break proof and update docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set deterministic PodGroup creation timestamps and assert JobOrderCompareFn tie conditions so task-order fallback behavior is validated consistently. Update BuildVictimsPriorityQueue comments to match current ordering paths, including orphaned jobs and queue-based handling. Signed-off-by: Hajnal Máté --- pkg/scheduler/framework/session_plugins.go | 12 ++++++++++-- .../framework/session_plugins_victim_order_test.go | 10 ++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/scheduler/framework/session_plugins.go b/pkg/scheduler/framework/session_plugins.go index 6fa126a8f64..0da35a3d293 100644 --- a/pkg/scheduler/framework/session_plugins.go +++ b/pkg/scheduler/framework/session_plugins.go @@ -656,6 +656,9 @@ func (ssn *Session) SubJobOrderFn(l, r interface{}) bool { return lv.UID < rv.UID } +// JobOrderCompareFn compares l and r by running enabled JobOrder plugins in +// order and returning the first non-zero comparison result. It returns 0 if +// all plugins consider l and r equal. func (ssn *Session) JobOrderCompareFn(l, r interface{}) int { for _, tier := range ssn.Tiers { for _, plugin := range tier.Plugins { @@ -1095,8 +1098,13 @@ func (ssn *Session) HyperNodeGradientForSubJobFn(subJob *api.SubJobInfo, hyperNo } // BuildVictimsPriorityQueue returns a priority queue with victims sorted by: -// if victims has same job id, sorted by !ssn.TaskOrderFn -// if victims has different job id, sorted by !ssn.JobOrderFn +// 1. If victims belong to the same job, use !ssn.TaskOrderFn. +// 2. If either victim's job is missing, evict orphaned tasks first; if both +// are orphaned, use !ssn.TaskOrderFn. +// 3. If the preemptor job is missing or victims are in the same queue, compare +// jobs with JobOrderCompareFn and use !ssn.TaskOrderFn as a tie-break. +// 4. If victims are in different queues and preemptor job exists, use +// ssn.VictimQueueOrderFn. func (ssn *Session) BuildVictimsPriorityQueue(victims []*api.TaskInfo, preemptor *api.TaskInfo) *util.PriorityQueue { jobThenTaskOrder := func(lvJob, rvJob *api.JobInfo, l, r interface{}) bool { if cmp := ssn.JobOrderCompareFn(lvJob, rvJob); cmp != 0 { diff --git a/pkg/scheduler/framework/session_plugins_victim_order_test.go b/pkg/scheduler/framework/session_plugins_victim_order_test.go index dbffe605035..fdaf747ee65 100644 --- a/pkg/scheduler/framework/session_plugins_victim_order_test.go +++ b/pkg/scheduler/framework/session_plugins_victim_order_test.go @@ -18,9 +18,11 @@ package framework_test import ( "testing" + "time" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" schedulingv1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" "volcano.sh/volcano/pkg/scheduler/api" @@ -39,6 +41,8 @@ func TestBuildVictimsPriorityQueueJobTieBreaksWithTaskOrder(t *testing.T) { nodeN1 := util.BuildNode("n1", api.BuildResourceList("10", "10Gi", []api.ScalarResource{{Name: "pods", Value: "20"}}...), nil) pgHigh := util.BuildPodGroup("pg-high", "ns1", "q1", 1, nil, schedulingv1.PodGroupRunning) pgLow := util.BuildPodGroup("pg-low", "ns1", "q1", 1, nil, schedulingv1.PodGroupRunning) + pgHigh.CreationTimestamp = metav1.NewTime(time.Unix(20, 0)) + pgLow.CreationTimestamp = metav1.NewTime(time.Unix(10, 0)) pgPreemptor := util.BuildPodGroup("pg-preemptor", "ns1", "q1", 1, nil, schedulingv1.PodGroupPending) pHigh := util.BuildPodWithPriority("ns1", "p-high", "n1", v1.PodRunning, api.BuildResourceList("1", "1Gi"), "pg-high", nil, nil, &highPri) pLow := util.BuildPodWithPriority("ns1", "p-low", "n1", v1.PodRunning, api.BuildResourceList("1", "1Gi"), "pg-low", nil, nil, &lowPri) @@ -92,6 +96,9 @@ func TestBuildVictimsPriorityQueueJobTieBreaksWithTaskOrder(t *testing.T) { assert.NotNil(t, high) assert.NotNil(t, low) assert.NotNil(t, preemptor) + highJob := ssn.Jobs[high.Job] + lowJob := ssn.Jobs[low.Job] + assert.Equal(t, 0, ssn.JobOrderCompareFn(highJob, lowJob)) victims := []*api.TaskInfo{high, low} pq := ssn.BuildVictimsPriorityQueue(victims, preemptor) @@ -108,6 +115,9 @@ func TestBuildVictimsPriorityQueueJobTieBreaksWithTaskOrder(t *testing.T) { low := findTask(ssn, "p-low") assert.NotNil(t, high) assert.NotNil(t, low) + highJob := ssn.Jobs[high.Job] + lowJob := ssn.Jobs[low.Job] + assert.Equal(t, 0, ssn.JobOrderCompareFn(highJob, lowJob)) victims := []*api.TaskInfo{high, low} pq := ssn.BuildVictimsPriorityQueue(victims, &api.TaskInfo{Job: api.JobID("missing")}) From 679eca74f9548b5994efdee0a324f5d44fb94d67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Wed, 18 Mar 2026 10:38:38 +0100 Subject: [PATCH 08/21] enhancement(capacity): block cyclic sibling reclaim in parent mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prevent sibling leaf reclaim loops when parentBasedReclaimEnabled is true and both leaves under the same parent have no relevant deserved signal for the reclaimer request. keep parent-level reclaim semantics unchanged for cross-parent paths. Signed-off-by: Hajnal Máté --- pkg/scheduler/plugins/capacity/capacity.go | 166 ++++++++++++++++----- 1 file changed, 130 insertions(+), 36 deletions(-) diff --git a/pkg/scheduler/plugins/capacity/capacity.go b/pkg/scheduler/plugins/capacity/capacity.go index 99d7548539f..bedb056b109 100644 --- a/pkg/scheduler/plugins/capacity/capacity.go +++ b/pkg/scheduler/plugins/capacity/capacity.go @@ -41,6 +41,8 @@ const ( // Using the name of the plugin will likely help us avoid collisions with other plugins. capacityStateKey = PluginName rootQueueID = "root" + // Holds the argument key of parentBasedReclaimEnabled + parentBasedReclaimEnabled = "parentBasedReclaimEnabled" ) type capacityPlugin struct { @@ -50,15 +52,18 @@ type capacityPlugin struct { queueOpts map[api.QueueID]*queueAttr // Arguments given for the plugin - pluginArguments framework.Arguments + pluginArguments framework.Arguments + parentBasedReclaimEnabled bool } type queueAttr struct { - queueID api.QueueID - name string - share float64 - ancestors []api.QueueID - children map[api.QueueID]*queueAttr + queueID api.QueueID + name string + share float64 + + ancestors []api.QueueID + parentQueueID api.QueueID + children map[api.QueueID]*queueAttr deserved *api.Resource allocated *api.Resource @@ -75,12 +80,16 @@ type queueAttr struct { // New return capacityPlugin action func New(arguments framework.Arguments) framework.Plugin { - return &capacityPlugin{ - totalResource: api.EmptyResource(), - totalGuarantee: api.EmptyResource(), - queueOpts: map[api.QueueID]*queueAttr{}, - pluginArguments: arguments, + // Create capacity plugin instance + capacityPlugin := &capacityPlugin{ + totalResource: api.EmptyResource(), + totalGuarantee: api.EmptyResource(), + queueOpts: map[api.QueueID]*queueAttr{}, + pluginArguments: arguments, + parentBasedReclaimEnabled: false, } + arguments.GetBool(&capacityPlugin.parentBasedReclaimEnabled, parentBasedReclaimEnabled) + return capacityPlugin } func (cp *capacityPlugin) Name() string { @@ -95,9 +104,12 @@ func (cp *capacityPlugin) OnSessionOpen(ssn *framework.Session) { hierarchyEnabled := ssn.HierarchyEnabled(cp.Name()) readyToSchedule := true + parentBasedReclaimEnabled := false if hierarchyEnabled { readyToSchedule = cp.buildHierarchicalQueueAttrs(ssn) - klog.V(4).Infof("Hierarchy is enabled in capacity plugin") + // parentBasedReclaimEnabled is true, only when the argument is set to true and hierarchy is enabled. + parentBasedReclaimEnabled = cp.parentBasedReclaimEnabled + klog.V(4).Infof("Hierarchy is enabled in capacity plugin, with parentBasedReclaim: %v", parentBasedReclaimEnabled) } else { cp.buildQueueAttrs(ssn) } @@ -109,8 +121,20 @@ func (cp *capacityPlugin) OnSessionOpen(ssn *framework.Session) { klog.V(3).Infof("Capacity plugin failed to check queue's hierarchical structure!") return victims, util.Reject } + reclaimerJob := ssn.Jobs[reclaimer.Job] + if reclaimerJob == nil { + klog.Warningf("[capacity] Skip reclaim: reclaimer <%s/%s> job <%s> not found in session", reclaimer.Namespace, reclaimer.Name, reclaimer.Job) + return victims, util.Reject + } + reclaimerAttr := cp.queueOpts[reclaimerJob.Queue] + if reclaimerAttr == nil { + klog.Warningf("[capacity] Skip reclaim: reclaimer queue <%s> not found in queueOpts", reclaimerJob.Queue) + return victims, util.Reject + } - for _, reclaimee := range reclaimees { + reclaimeesQueue := ssn.BuildVictimsPriorityQueue(reclaimees, reclaimer) + for !reclaimeesQueue.Empty() { + reclaimee := reclaimeesQueue.Pop().(*api.TaskInfo) job := ssn.Jobs[reclaimee.Job] if job == nil { klog.Warningf("[capacity] Skip reclaimee <%s/%s>: job <%s> not found in session (orphaned task from deleted PodGroup)", @@ -133,6 +157,19 @@ func (cp *capacityPlugin) OnSessionOpen(ssn *framework.Session) { klog.V(5).Infof("%s, skip it.", reason) continue } + parentCheckNeeded := parentBasedReclaimEnabled && + attr.parentQueueID != "" && + attr.parentQueueID != rootQueueID && + attr.parentQueueID != reclaimerAttr.parentQueueID + var parentAttr *queueAttr + if parentCheckNeeded { + parentAttr = cp.queueOpts[attr.parentQueueID] + if parentAttr == nil { + klog.Warningf("[capacity] Skip reclaimee <%s/%s>: parent queue <%s> not found in queueOpts", + reclaimee.Namespace, reclaimee.Name, attr.parentQueueID) + continue + } + } // allocations maps each queue to its current allocated resources (cloned) and 'allocated' points to this resource object. // As victims (reclaimees) are selected, their resource requests are subtracted from the corresponding queue's allocation via this pointer. @@ -141,34 +178,71 @@ func (cp *capacityPlugin) OnSessionOpen(ssn *framework.Session) { allocations[job.Queue] = attr.allocated.Clone() } allocated := allocations[job.Queue] + var parentAllocated *api.Resource + if parentCheckNeeded { + if _, found := allocations[attr.parentQueueID]; !found { + allocations[attr.parentQueueID] = parentAttr.allocated.Clone() + } + parentAllocated = allocations[attr.parentQueueID] + } // Check guarantee if satisfies, _ := cp.checkGuaranteeConstraint(allocated, reclaimee, attr.guarantee); !satisfies { continue } - // If the reclaimee has no intersecting resource dimensions with deserved, it is a victim. + childEligible := false if isVictim, reason := cp.isImmediateVictim(reclaimee, attr.deserved); isVictim { - allocated.Sub(reclaimee.Resreq) - victims = append(victims, reclaimee) - klog.V(5).Infof("%s. It's a victim. Current victims: %+v.", reason, victims) - continue - } - - // Check deserved - if exceeds, dims, reason := cp.checkDeservedExceedance( - allocated, attr.deserved, reclaimee, reclaimer, attr.name); !exceeds { - klog.V(5).Infof("%s", reason) - continue - } else { + if hierarchyEnabled && parentBasedReclaimEnabled && + attr.parentQueueID != "" && attr.parentQueueID != rootQueueID && + attr.parentQueueID == reclaimerAttr.parentQueueID && + !hasRelevantDeserved(reclaimer, reclaimerAttr.deserved) { + klog.V(5).Infof("[capacity] Skip reclaim for reclaimee <%s/%s> from queue <%s>: sibling queues share parent <%s> and reclaimer leaf queue has no relevant deserved signal", + reclaimee.Namespace, reclaimee.Name, attr.queueID, attr.parentQueueID) + continue + } + childEligible = true + klog.V(5).Infof("%s. It's a victim for queue <%s>.", reason, attr.name) + } else if exceeds, dims, reason := cp.checkDeservedExceedance( + allocated, attr.deserved, reclaimee, reclaimer, attr.name); exceeds { + childEligible = true klog.V(5).Infof("[capacity] Reclaimee <%s/%s> is a victim from queue <%s> for reclaimer <%s/%s>. "+ "Allocated: <%v>, Deserved: <%v>, Reclaimee Resreq: <%v>, Reclaimable on dimensions: %v.", reclaimee.Namespace, reclaimee.Name, attr.queueID, reclaimer.Namespace, reclaimer.Name, allocated, attr.deserved, reclaimee.Resreq, dims) - allocated.Sub(reclaimee.Resreq) - victims = append(victims, reclaimee) - klog.V(5).Infof("[capacity] Current victims: %+v.", victims) + } else { + klog.V(5).Infof("%s.", reason) + } + + if !childEligible { + continue + } + + if parentCheckNeeded { + parentEligible := false + if isVictim, reasonParent := cp.isImmediateVictim(reclaimee, parentAttr.deserved); isVictim { + parentEligible = true + klog.V(5).Infof("%s. It's a victim for parent queue <%s>.", reasonParent, parentAttr.name) + } else if exceeds, parentDims, parentReason := cp.checkDeservedExceedance( + parentAllocated, parentAttr.deserved, reclaimee, reclaimer, parentAttr.name); exceeds { + parentEligible = true + klog.V(5).Infof("[capacity] Reclaimee <%s/%s> is a victim from parent queue <%s> for reclaimer <%s/%s>. "+ + "Allocated: <%v>, Deserved: <%v>, Reclaimee Resreq: <%v>, Reclaimable on dimensions: %v.", + reclaimee.Namespace, reclaimee.Name, parentAttr.name, reclaimer.Namespace, reclaimer.Name, + parentAllocated, parentAttr.deserved, reclaimee.Resreq, parentDims) + } else { + klog.V(5).Infof("%s.", parentReason) + } + + if !parentEligible { + continue + } + parentAllocated.Sub(reclaimee.Resreq) } + + allocated.Sub(reclaimee.Resreq) + victims = append(victims, reclaimee) + klog.V(5).Infof("[capacity] Current victims: %+v.", victims) } klog.V(4).Infof("[capacity] Victims from capacity plugin: victims=%+v reclaimer=%s.", victims, reclaimer) return victims, util.Permit @@ -204,8 +278,23 @@ func (cp *capacityPlugin) OnSessionOpen(ssn *framework.Session) { "The futureUsed: %v, deserved: %v, allocated: %v, task requested: %v", queue.Name, resourceNames, futureUsed, attr.deserved, attr.allocated, task.Resreq) } else { - klog.V(3).Infof("Queue <%v> can not reclaim, futureUsed: %v, deserved: %v, requested: %v", + klog.V(4).Infof("Queue <%v> itself can not reclaim, futureUsed: %v, deserved: %v, requested: %v", queue.Name, futureUsed, attr.deserved, task.Resreq) + // If parentBasedReclaimEnabled is true, check whether the direct parent can reclaim. + if parentBasedReclaimEnabled && attr.parentQueueID != "" && attr.parentQueueID != rootQueueID { + parentAttr := cp.queueOpts[attr.parentQueueID] + futureUsedParent := parentAttr.allocated.Clone().Add(task.Resreq) + isPreemptive, resourceNames = futureUsedParent.LessEqualPartlyWithDimensionZeroFiltered(parentAttr.deserved, task.Resreq) + if isPreemptive { + klog.V(3).Infof("Queue's parent <%v> can reclaim on resource dimensions: %v. "+ + "The futureUsedParent: %v, deserved: %v, allocated: %v, task requested: %v", + parentAttr.name, resourceNames, futureUsedParent, parentAttr.deserved, parentAttr.allocated, task.Resreq) + } else { + klog.V(4).Infof("Queue <%v> and its parent <%v> can not reclaim. "+ + "The futureUsedParent: %v, parentDeserved: %v, requested: %v", + queue.Name, parentAttr.name, futureUsedParent, parentAttr.deserved, task.Resreq) + } + } } // PreemptiveFn is the opposite of OverusedFn in proportion plugin cause as long as there is a one-dimensional @@ -769,10 +858,11 @@ func (cp *capacityPlugin) buildHierarchicalQueueAttrs(ssn *framework.Session) bo func (cp *capacityPlugin) newQueueAttr(queue *api.QueueInfo) *queueAttr { attr := &queueAttr{ - queueID: queue.UID, - name: queue.Name, - ancestors: make([]api.QueueID, 0), - children: make(map[api.QueueID]*queueAttr), + queueID: queue.UID, + name: queue.Name, + parentQueueID: api.QueueID(queue.Queue.Spec.Parent), + ancestors: make([]api.QueueID, 0), + children: make(map[api.QueueID]*queueAttr), deserved: api.NewResource(queue.Queue.Spec.Deserved), allocated: api.EmptyResource(), @@ -1145,6 +1235,10 @@ func (cp *capacityPlugin) isImmediateVictim( return false, "" } +func hasRelevantDeserved(reclaimer *api.TaskInfo, deserved *api.Resource) bool { + return len(api.Intersection(reclaimer.InitResreq, deserved)) > 0 +} + // checkDeservedExceedance checks if the queue's allocated resources exceed its deserved resources // on dimensions relevant to the reclaimee, making the reclaimee a valid victim. // Returns true if exceeds, along with the relevant dimensions and a reason message. @@ -1159,9 +1253,9 @@ func (cp *capacityPlugin) checkDeservedExceedance( if !reclaimable { reason := fmt.Sprintf( "[capacity] Queue <%v> allocated resources are not greater than deserved on any relevant dimension of reclaimee. "+ - "Hence reclaimee <%s/%s> cannot be reclaimed for reclaimer <%s/%s>. "+ + "Hence reclaimee <%s/%s> cannot be reclaimed for reclaimer <%s/%s> for queue <%s>. "+ "Deserved: <%v>, Allocated: <%v>, Reclaimee Resreq: <%v>", - queueName, reclaimee.Namespace, reclaimee.Name, reclaimer.Namespace, reclaimer.Name, deserved, allocated, reclaimee.Resreq, + queueName, reclaimee.Namespace, reclaimee.Name, reclaimer.Namespace, reclaimer.Name, queueName, deserved, allocated, reclaimee.Resreq, ) return false, nil, reason } From 8c24da29c278582afb5f6f1045ebed1b49e48901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Wed, 18 Mar 2026 11:33:49 +0100 Subject: [PATCH 09/21] test(capacity): add parent-based reclaim scenario suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add Test_capacityPlugin_ParentBasedReclaimScenarios with case1-case5 coverage for parent-based reclaim behavior under hierarchy. restore Test_capacityPlugin_OnSessionOpenWithHierarchy to baseline scope and keep parent-based scenarios in a dedicated test suite. Signed-off-by: Hajnal Máté --- .../plugins/capacity/capacity_test.go | 228 +++++++++++++++++- 1 file changed, 227 insertions(+), 1 deletion(-) diff --git a/pkg/scheduler/plugins/capacity/capacity_test.go b/pkg/scheduler/plugins/capacity/capacity_test.go index c67d517c1e5..8eacdc29077 100644 --- a/pkg/scheduler/plugins/capacity/capacity_test.go +++ b/pkg/scheduler/plugins/capacity/capacity_test.go @@ -34,6 +34,7 @@ import ( "volcano.sh/volcano/pkg/scheduler/framework" "volcano.sh/volcano/pkg/scheduler/plugins/gang" "volcano.sh/volcano/pkg/scheduler/plugins/predicates" + "volcano.sh/volcano/pkg/scheduler/plugins/priority" "volcano.sh/volcano/pkg/scheduler/uthelper" "volcano.sh/volcano/pkg/scheduler/util" ) @@ -496,7 +497,7 @@ func TestEnqueueAndAllocatable(t *testing.T) { } func Test_capacityPlugin_OnSessionOpenWithHierarchy(t *testing.T) { - plugins := map[string]framework.PluginBuilder{PluginName: New, predicates.PluginName: predicates.New, gang.PluginName: gang.New} + plugins := map[string]framework.PluginBuilder{PluginName: New, predicates.PluginName: predicates.New, gang.PluginName: gang.New, priority.PluginName: priority.New} trueValue := true actions := []framework.Action{enqueue.New(), reclaim.New(), allocate.New()} @@ -646,6 +647,7 @@ func Test_capacityPlugin_OnSessionOpenWithHierarchy(t *testing.T) { p17 := util.BuildPod("ns1", "p17", "n3", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "2"}, {Name: "rdma/hca", Value: "1"}}...), "pg17", make(map[string]string), map[string]string{}) p18 := util.BuildPod("ns1", "p18", "n3", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "2"}, {Name: "rdma/hca", Value: "1"}}...), "pg18", map[string]string{schedulingv1beta1.PodPreemptable: "false"}, map[string]string{}) + // Test case tests := []uthelper.TestCommonStruct{ { Name: "case0: Pod allocatable when queue is leaf queue", @@ -808,6 +810,11 @@ func Test_capacityPlugin_OnSessionOpenWithHierarchy(t *testing.T) { Name: predicates.PluginName, EnabledPredicate: &trueValue, }, + { + Name: priority.PluginName, + EnabledJobOrder: &trueValue, + EnabledTaskOrder: &trueValue, + }, { Name: gang.PluginName, EnabledJobStarving: &trueValue, @@ -827,6 +834,225 @@ func Test_capacityPlugin_OnSessionOpenWithHierarchy(t *testing.T) { } } +func Test_capacityPlugin_ParentBasedReclaimScenarios(t *testing.T) { + plugins := map[string]framework.PluginBuilder{ + PluginName: New, + predicates.PluginName: predicates.New, + gang.PluginName: gang.New, + priority.PluginName: priority.New, + } + actions := []framework.Action{enqueue.New(), reclaim.New(), allocate.New()} + + type scenario struct { + name string + includePriority bool + build func(t *testing.T) (nodes []*corev1.Node, pods []*corev1.Pod, pgs []*schedulingv1beta1.PodGroup, queues []*schedulingv1beta1.Queue) + expectPipeLined map[string][]string + expectEvicted []string + expectEvictNum int + } + + buildTier := func(includePriority bool) []conf.Tier { + trueValue := true + pluginOptions := []conf.PluginOption{ + { + Name: PluginName, + EnabledAllocatable: &trueValue, + EnablePreemptive: &trueValue, + EnabledReclaimable: &trueValue, + EnabledQueueOrder: &trueValue, + EnabledHierarchy: &trueValue, + EnabledJobEnqueued: &trueValue, + Arguments: framework.Arguments{ + "parentBasedReclaimEnabled": true, + }, + }, + { + Name: predicates.PluginName, + EnabledPredicate: &trueValue, + }, + { + Name: gang.PluginName, + EnabledJobStarving: &trueValue, + }, + } + + if includePriority { + pluginOptions = append(pluginOptions, conf.PluginOption{ + Name: priority.PluginName, + EnabledJobOrder: &trueValue, + EnabledTaskOrder: &trueValue, + }) + } + + return []conf.Tier{{Plugins: pluginOptions}} + } + + buildProjectQueueTree := func() (*schedulingv1beta1.Queue, *schedulingv1beta1.Queue, *schedulingv1beta1.Queue, *schedulingv1beta1.Queue, *schedulingv1beta1.Queue, *schedulingv1beta1.Queue, *schedulingv1beta1.Queue) { + root := buildQueueWithParents("root", "", api.BuildResourceList("16", "16Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "4"}, {Name: "rdma", Value: "1000"}}...), nil) + project1RootQueue := buildQueueWithParents("project1_root", "root", api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}}...), api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "2"}}...)) + project1NonPreemptableQueue := buildQueueWithParents("project1_non-preemptable", "project1_root", api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}}...), api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}}...)) + project1PreemptableQueue := buildQueueWithParents("project1_preemptable", "project1_root", nil, nil) + project2RootQueue := buildQueueWithParents("project2_root", "root", api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "3"}}...), api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "4"}}...)) + project2NonPreemptableQueue := buildQueueWithParents("project2_non-preemptable", "project2_root", api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "3"}}...), api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "3"}}...)) + project2PreemptableQueue := buildQueueWithParents("project2_preemptable", "project2_root", nil, nil) + return root, project1RootQueue, project1NonPreemptableQueue, project1PreemptableQueue, project2RootQueue, project2NonPreemptableQueue, project2PreemptableQueue + } + + scenarios := []scenario{ + { + name: "case1: Can reclaim based on parent deserved when parentBasedReclaimEnabled is true", + includePriority: true, + build: func(t *testing.T) (nodes []*corev1.Node, pods []*corev1.Pod, pgs []*schedulingv1beta1.PodGroup, queues []*schedulingv1beta1.Queue) { + n1 := util.BuildNode("n1", api.BuildResourceList("16", "16Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "4"}, {Name: "rdma/hca", Value: "1001"}, {Name: "pods", Value: "11"}}...), map[string]string{}) + root := buildQueueWithParents("root", "", api.BuildResourceList("16", "16Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "4"}, {Name: "rdma", Value: "1000"}}...), nil) + case1Queue1 := buildQueueWithParents("case1_queue1", "root", api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}}...), nil) + case1Queue11 := buildQueueWithParents("case1_queue11", "case1_queue1", nil, nil) + case1Queue2 := buildQueueWithParents("case1_queue2", "root", nil, nil) + + pg1 := util.BuildPodGroup("pg1", "ns1", "case1_queue2", 1, nil, schedulingv1beta1.PodGroupRunning) + pg2 := util.BuildPodGroup("pg2", "ns1", "case1_queue11", 1, nil, schedulingv1beta1.PodGroupInqueue) + + p1 := util.BuildPod("ns1", "p1", "n1", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "4"}, {Name: "rdma/hca", Value: "1"}}...), "pg1", map[string]string{}, map[string]string{}) + p2 := util.BuildPod("ns1", "p2", "", corev1.PodPending, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg2", map[string]string{}, map[string]string{}) + + return []*corev1.Node{n1}, []*corev1.Pod{p1, p2}, []*schedulingv1beta1.PodGroup{pg1, pg2}, []*schedulingv1beta1.Queue{root, case1Queue1, case1Queue11, case1Queue2} + }, + expectPipeLined: map[string][]string{"ns1/pg2": {"n1"}}, + expectEvicted: []string{"ns1/p1"}, + expectEvictNum: 1, + }, + { + name: "case2: Cross-parent reclaim pipelines project1 pending job when project2 child and parent are both over deserved", + includePriority: true, + build: func(t *testing.T) (nodes []*corev1.Node, pods []*corev1.Pod, pgs []*schedulingv1beta1.PodGroup, queues []*schedulingv1beta1.Queue) { + n1 := util.BuildNode("n1", api.BuildResourceList("16", "16Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "4"}, {Name: "rdma/hca", Value: "1001"}, {Name: "pods", Value: "11"}}...), map[string]string{}) + root, project1RootQueue, project1NonPreemptableQueue, project1PreemptableQueue, project2RootQueue, project2NonPreemptableQueue, project2PreemptableQueue := buildProjectQueueTree() + + pg1 := util.BuildPodGroup("pg1", "ns1", "project2_preemptable", 1, nil, schedulingv1beta1.PodGroupRunning) + pg2 := util.BuildPodGroup("pg2", "ns1", "project2_preemptable", 1, nil, schedulingv1beta1.PodGroupRunning) + pg3 := util.BuildPodGroup("pg3", "ns1", "project2_preemptable", 1, nil, schedulingv1beta1.PodGroupRunning) + pg4 := util.BuildPodGroup("pg4", "ns1", "project2_preemptable", 1, nil, schedulingv1beta1.PodGroupRunning) + pg5 := util.BuildPodGroup("pg5", "ns1", "project1_preemptable", 1, nil, schedulingv1beta1.PodGroupPending) + + priority1, priority2 := int32(1), int32(2) + p1 := util.BuildPodWithPriority("ns1", "p1", "n1", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg1", map[string]string{}, map[string]string{}, &priority2) + p2 := util.BuildPodWithPriority("ns1", "p2", "n1", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg2", map[string]string{}, map[string]string{}, &priority2) + p3 := util.BuildPodWithPriority("ns1", "p3", "n1", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg3", map[string]string{}, map[string]string{}, &priority2) + p4 := util.BuildPodWithPriority("ns1", "p4", "n1", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg4", map[string]string{}, map[string]string{}, &priority1) + p5 := util.BuildPod("ns1", "p5", "", corev1.PodPending, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg5", map[string]string{}, map[string]string{}) + + return []*corev1.Node{n1}, []*corev1.Pod{p1, p2, p3, p4, p5}, []*schedulingv1beta1.PodGroup{pg1, pg2, pg3, pg4, pg5}, []*schedulingv1beta1.Queue{root, project1RootQueue, project1NonPreemptableQueue, project1PreemptableQueue, project2RootQueue, project2NonPreemptableQueue, project2PreemptableQueue} + }, + expectPipeLined: map[string][]string{"ns1/pg5": {"n1"}}, + expectEvicted: []string{"ns1/p4"}, + expectEvictNum: 1, + }, + { + name: "case3: Cross-parent reclaim can evict sibling-queue victim first", + includePriority: true, + build: func(t *testing.T) (nodes []*corev1.Node, pods []*corev1.Pod, pgs []*schedulingv1beta1.PodGroup, queues []*schedulingv1beta1.Queue) { + n1 := util.BuildNode("n1", api.BuildResourceList("16", "16Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "4"}, {Name: "rdma/hca", Value: "1001"}, {Name: "pods", Value: "11"}}...), map[string]string{}) + root, project1RootQueue, project1NonPreemptableQueue, project1PreemptableQueue, project2RootQueue, project2NonPreemptableQueue, project2PreemptableQueue := buildProjectQueueTree() + + pg1 := util.BuildPodGroup("pg1", "ns1", "project2_preemptable", 1, nil, schedulingv1beta1.PodGroupRunning) + pg2 := util.BuildPodGroup("pg2", "ns1", "project2_preemptable", 1, nil, schedulingv1beta1.PodGroupRunning) + pg3 := util.BuildPodGroup("pg3", "ns1", "project2_preemptable", 1, nil, schedulingv1beta1.PodGroupRunning) + pg4Reclaimed := util.BuildPodGroup("pg4_reclaimed", "ns1", "project2_preemptable", 1, nil, schedulingv1beta1.PodGroupInqueue) + pg5Pipelined := util.BuildPodGroup("pg5_pipelined", "ns1", "project1_preemptable", 1, nil, schedulingv1beta1.PodGroupRunning) + pg6 := util.BuildPodGroup("pg6", "ns1", "project2_non-preemptable", 1, nil, schedulingv1beta1.PodGroupPending) + + priority2 := int32(2) + p1 := util.BuildPodWithPriority("ns1", "p1", "n1", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg1", map[string]string{}, map[string]string{}, &priority2) + p2 := util.BuildPodWithPriority("ns1", "p2", "n1", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg2", map[string]string{}, map[string]string{}, &priority2) + p3 := util.BuildPodWithPriority("ns1", "p3", "n1", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg3", map[string]string{}, map[string]string{}, &priority2) + p4Reclaimed := util.BuildPod("ns1", "p4", "", corev1.PodPending, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg4_reclaimed", map[string]string{}, map[string]string{}) + p5Pipelined := util.BuildPod("ns1", "p5", "n1", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg5_pipelined", map[string]string{}, map[string]string{}) + p6 := util.BuildPod("ns1", "p6", "", corev1.PodPending, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg6", map[string]string{}, map[string]string{}) + + return []*corev1.Node{n1}, []*corev1.Pod{p1, p2, p3, p4Reclaimed, p5Pipelined, p6}, []*schedulingv1beta1.PodGroup{pg1, pg2, pg3, pg4Reclaimed, pg5Pipelined, pg6}, []*schedulingv1beta1.Queue{root, project1RootQueue, project1NonPreemptableQueue, project1PreemptableQueue, project2RootQueue, project2NonPreemptableQueue, project2PreemptableQueue} + }, + expectPipeLined: map[string][]string{"ns1/pg6": {"n1"}}, + expectEvicted: []string{"ns1/p3"}, + expectEvictNum: 1, + }, + { + name: "case4: Cross-parent reclaim is blocked when victim child is not over deserved", + includePriority: true, + build: func(t *testing.T) (nodes []*corev1.Node, pods []*corev1.Pod, pgs []*schedulingv1beta1.PodGroup, queues []*schedulingv1beta1.Queue) { + n1 := util.BuildNode("n1", api.BuildResourceList("8", "8Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1001"}, {Name: "pods", Value: "11"}}...), map[string]string{}) + + root := buildQueueWithParents("root", "", api.BuildResourceList("16", "16Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "4"}, {Name: "rdma", Value: "1000"}}...), nil) + case4Parent1 := buildQueueWithParents("case4_parent1", "root", api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}}...), nil) + case4Child1 := buildQueueWithParents("case4_child1", "case4_parent1", api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}}...), nil) + case4Child1Sibling := buildQueueWithParents("case4_child1_sibling", "case4_parent1", nil, nil) + case4Parent2 := buildQueueWithParents("case4_parent2", "root", api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}}...), nil) + case4Child2 := buildQueueWithParents("case4_child2", "case4_parent2", api.BuildResourceList("", "", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}}...), nil) + + pg1 := util.BuildPodGroup("pg1", "ns1", "case4_child1", 1, nil, schedulingv1beta1.PodGroupRunning) + pg2 := util.BuildPodGroup("pg2", "ns1", "case4_child2", 1, nil, schedulingv1beta1.PodGroupPending) + pg3 := util.BuildPodGroup("pg3", "ns1", "case4_child1_sibling", 1, nil, schedulingv1beta1.PodGroupRunning) + + p1 := util.BuildPod("ns1", "p1", "n1", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg1", map[string]string{}, map[string]string{}) + p2 := util.BuildPod("ns1", "p2", "", corev1.PodPending, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg2", map[string]string{}, map[string]string{}) + p3 := util.BuildPod("ns1", "p3", "n1", corev1.PodRunning, api.BuildResourceList("1", "1Gi", []api.ScalarResource{{Name: "nvidia.com/a100", Value: "1"}, {Name: "rdma/hca", Value: "1"}}...), "pg3", map[string]string{schedulingv1beta1.PodPreemptable: "false"}, map[string]string{}) + + return []*corev1.Node{n1}, []*corev1.Pod{p1, p2, p3}, []*schedulingv1beta1.PodGroup{pg1, pg2, pg3}, []*schedulingv1beta1.Queue{root, case4Parent1, case4Child1, case4Child1Sibling, case4Parent2, case4Child2} + }, + expectPipeLined: map[string][]string{}, + expectEvicted: []string{}, + expectEvictNum: 0, + }, + { + name: "case5: Parent-based reclaim should be blocked when leaf deserved is unset for sibling queues", + includePriority: false, + build: func(t *testing.T) (nodes []*corev1.Node, pods []*corev1.Pod, pgs []*schedulingv1beta1.PodGroup, queues []*schedulingv1beta1.Queue) { + n1 := util.BuildNode("n1", api.BuildResourceList("2", "2Gi", []api.ScalarResource{{Name: "pods", Value: "10"}}...), map[string]string{}) + root := buildQueueWithParents("root", "", nil, nil) + parent := buildQueueWithParents("parent", "root", api.BuildResourceList("4", "4Gi"), api.BuildResourceList("4", "4Gi")) + queueA := buildQueueWithParents("queue-a", "parent", nil, nil) + queueB := buildQueueWithParents("queue-b", "parent", nil, nil) + + pgVictim := util.BuildPodGroup("pg-victim", "ns1", "queue-b", 1, nil, schedulingv1beta1.PodGroupRunning) + pgReclaimer := util.BuildPodGroup("pg-reclaimer", "ns1", "queue-a", 1, nil, schedulingv1beta1.PodGroupInqueue) + + pVictim := util.BuildPod("ns1", "p-victim", "n1", corev1.PodRunning, api.BuildResourceList("2", "2Gi"), "pg-victim", nil, nil) + pReclaimer := util.BuildPod("ns1", "p-reclaimer", "", corev1.PodPending, api.BuildResourceList("2", "2Gi"), "pg-reclaimer", nil, nil) + + return []*corev1.Node{n1}, []*corev1.Pod{pVictim, pReclaimer}, []*schedulingv1beta1.PodGroup{pgVictim, pgReclaimer}, []*schedulingv1beta1.Queue{root, parent, queueA, queueB} + }, + expectPipeLined: map[string][]string{}, + expectEvicted: []string{}, + expectEvictNum: 0, + }, + } + + for i, sc := range scenarios { + t.Run(sc.name, func(t *testing.T) { + nodes, pods, pgs, queues := sc.build(t) + test := uthelper.TestCommonStruct{ + Name: sc.name, + Plugins: plugins, + Nodes: nodes, + Pods: pods, + PodGroups: pgs, + Queues: queues, + ExpectPipeLined: sc.expectPipeLined, + ExpectEvicted: sc.expectEvicted, + ExpectEvictNum: sc.expectEvictNum, + } + + tiers := buildTier(sc.includePriority) + test.RegisterSession(tiers, nil) + defer test.Close() + test.Run(actions) + if err := test.CheckAll(i); err != nil { + t.Fatal(err) + } + }) + } +} + func buildQueueWithParents(name string, parent string, deserved corev1.ResourceList, cap corev1.ResourceList) *schedulingv1beta1.Queue { queue := util.BuildQueueWithResourcesQuantity(name, deserved, cap) queue.Spec.Parent = parent From 68c772023a884362641bf3003fba7bc29f20db76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Wed, 18 Mar 2026 11:33:49 +0100 Subject: [PATCH 10/21] docs(capacity): add parent-based reclaim scenario diagrams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit document case1-case5 from Test_capacityPlugin_ParentBasedReclaimScenarios with queue topology diagrams, workload details, and expected reclaim outcomes. reorder key-function section so updateParentQueue and note placement match the design structure. Signed-off-by: Hajnal Máté --- .../hierarchical-queue-on-capacity-plugin.md | 119 +++++++++++++++++- 1 file changed, 117 insertions(+), 2 deletions(-) diff --git a/docs/design/hierarchical-queue-on-capacity-plugin.md b/docs/design/hierarchical-queue-on-capacity-plugin.md index ee81eadd101..045e96dec07 100644 --- a/docs/design/hierarchical-queue-on-capacity-plugin.md +++ b/docs/design/hierarchical-queue-on-capacity-plugin.md @@ -36,6 +36,16 @@ For example, consider the following hierarchical queue structure and jobs: When job A performs reclaim or preempt actions, the system should first consider job B (belonging to the same parent queue a), and then consider job C (belonging to the same parent queue root). +### Story 5 + +When `parentBasedReclaimEnabled` is enabled for the capacity plugin and hierarchy is enabled, cross-parent reclaim should respect parent-level deserved resources. + +If a child queue is over its own deserved but its parent queue is still below parent deserved, the child's extra usage is treated as intra-parent borrowing and should not be reclaimed by other parent trees. + +Reclaim from that child becomes eligible only when both the child queue and the relevant parent queue are over deserved on reclaim-relevant dimensions. + +For sibling reclaim under the same direct parent, existing child-level reclaim checks remain in effect. + ## Design detail ### Webhook @@ -124,11 +134,116 @@ When job A performs reclaim or preempt actions, the system should first consider - `VictimTaskOrderFn`: Prioritize tasks that belong to the same parent queue as the preemptor. If necessary, it will consider other tasks from the bottom up in the queue hierarchy. - `VictimJobOrderFn`: Similar to `VictimTaskOrderFn`, prioritize jobs that belong to the same parent queue as the preemptor. If necessary, it will consider other jobs from the bottom up in the queue hierarchy. - - `updateParentQueue`: Update the resource status of parent queues whenever the `updateShare` function is executed. It will traverse the queue hierarchy from the bottom up and update the resource information of parent queues accordingly. **Note:** The above modifications are primarily applicable when `EnabledHierarchy` is set to true. If the capacity plugin does not require hierarchical queue management, the existing implementations of these functions will be retained. +#### Configuration example + +Enable hierarchy and parent-based reclaim in scheduler configuration: + +```yaml +actions: "enqueue, allocate, backfill" +tiers: + - plugins: + - name: capacity + enableHierarchy: true + arguments: + parentBasedReclaimEnabled: true + - name: gang + - name: priority +``` + +#### Unit test scenarios and behavior map + +The table-driven test `Test_capacityPlugin_ParentBasedReclaimScenarios` covers case1-case5 with explicit queue topology and reclaim behavior when `parentBasedReclaimEnabled=true`. + +**case1: Can reclaim based on parent deserved when parentBasedReclaimEnabled is true** + +```mermaid +graph TD + R[root] + Q1[case1_queue1
deserved: a100=1
capability: unset] + Q11[case1_queue11
deserved: unset
capability: unset] + Q2[case1_queue2
deserved: unset
capability: unset] + R --> Q1 + Q1 --> Q11 + R --> Q2 +``` + +- Workloads: `p1` running on `n1` in `case1_queue2` requests `a100=4`; `p2` pending in `case1_queue11` requests `a100=1`. +- Setting: `parentBasedReclaimEnabled=true`. +- Expected: `p1` is evicted and `p2` (podgroup `pg2`) is pipelined to `n1`. + +**case2: Cross-parent reclaim pipelines project1 pending job when project2 child and parent are both over deserved** + +```mermaid +graph TD + R[root] + P1[project1_root
deserved: a100=1
capability: a100=2] + P1NP[project1_non-preemptable
deserved: a100=1
capability: a100=1] + P1P[project1_preemptable
deserved: unset
capability: unset] + P2[project2_root
deserved: a100=3
capability: a100=4] + P2NP[project2_non-preemptable
deserved: a100=3
capability: a100=3] + P2P[project2_preemptable
deserved: unset
capability: unset] + R --> P1 + P1 --> P1NP + P1 --> P1P + R --> P2 + P2 --> P2NP + P2 --> P2P +``` + +- Workloads: `p1..p4` running on `n1` in `project2_preemptable` (`a100=1` each), `p5` pending in `project1_preemptable` (`a100=1`). +- Setting: `parentBasedReclaimEnabled=true`. +- Expected: lower-priority `p4` is evicted and `p5` (podgroup `pg5`) is pipelined to `n1`. + +**case3: Cross-parent reclaim can evict sibling-queue victim first** + +Queue topology is identical to case2. + +- Workloads: continuation-style state with `p1..p3` running on `n1` in `project2_preemptable` (`a100=1` each), `p4` pending as reclaimed (`a100=1`), `p5` already running in `project1_preemptable` (`a100=1`), and `p6` pending in `project2_non-preemptable` (`a100=1`). +- Setting: `parentBasedReclaimEnabled=true`. +- Expected: `p3` is evicted and `p6` (podgroup `pg6`) is pipelined to `n1`. + +**case4: Cross-parent reclaim is blocked when victim child is not over deserved** + +```mermaid +graph TD + R[root] + P1[case4_parent1
deserved: a100=1
capability: unset] + C1[case4_child1
deserved: a100=1
capability: unset] + C1S[case4_child1_sibling
deserved: unset
capability: unset] + P2[case4_parent2
deserved: a100=1
capability: unset] + C2[case4_child2
deserved: a100=1
capability: unset] + R --> P1 + P1 --> C1 + P1 --> C1S + R --> P2 + P2 --> C2 +``` + +- Workloads: `p1` running on `n1` in `case4_child1` (`a100=1`), `p3` running in sibling queue (non-preemptable, `a100=1`), `p2` pending in `case4_child2` (`a100=1`). +- Setting: `parentBasedReclaimEnabled=true`. +- Expected: no eviction and no pipeline. + +**case5: Parent-based reclaim is blocked when sibling leaves have no relevant deserved signal** + +```mermaid +graph TD + R[root] + P[parent
deserved: cpu=4, mem=4Gi
capability: cpu=4, mem=4Gi] + QA[queue-a
deserved: unset
capability: unset] + QB[queue-b
deserved: unset
capability: unset] + R --> P + P --> QA + P --> QB +``` + +- Workloads: `p-victim` running on `n1` in `queue-b` (`cpu=2, mem=2Gi`), `p-reclaimer` pending in `queue-a` (`cpu=2, mem=2Gi`). +- Setting: `parentBasedReclaimEnabled=true`. +- Expected: no eviction and no pipeline. + ### Vcctl - Design relevant vcctl commands, such as commands to obtain the child queues of a specific queue or commands to retrieve the entire hierarchical queue structure. @@ -137,4 +252,4 @@ When job A performs reclaim or preempt actions, the system should first consider - The current design does not support scheduling jobs/podgroups to non-leaf queues. Only leaf queues can directly schedule and allocate resources to jobs/podgroups. - The current design does not consider the queue migration issues related to hierarchical queue management in order to avoid manual operation and maintenance. -- When introducing a paused scheduling state for queues, optimize the management of the hierarchical queue structure under this state. For example, if a parent queue is paused for scheduling, it will cause its child queues to be paused as well. When resuming the parent queue, provide the capability to automatically resume the child queues in conjunction. \ No newline at end of file +- When introducing a paused scheduling state for queues, optimize the management of the hierarchical queue structure under this state. For example, if a parent queue is paused for scheduling, it will cause its child queues to be paused as well. When resuming the parent queue, provide the capability to automatically resume the child queues in conjunction. From 48f78691bc7eac75ed17df4ca393e50d617bc866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Mon, 15 Dec 2025 10:35:47 +0100 Subject: [PATCH 11/21] Group eviction support in statement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Hajnal Máté --- pkg/scheduler/framework/statement.go | 61 +++++++++- pkg/scheduler/framework/statement_test.go | 130 +++++++++++++++++++++- 2 files changed, 186 insertions(+), 5 deletions(-) diff --git a/pkg/scheduler/framework/statement.go b/pkg/scheduler/framework/statement.go index e3c279722ba..74ab5dd4a67 100644 --- a/pkg/scheduler/framework/statement.go +++ b/pkg/scheduler/framework/statement.go @@ -44,6 +44,10 @@ const ( Allocate ) +const ( + GroupEvictionPolicyAnnotationKey = "volcano.sh/group-eviction-policy" +) + type operation struct { name Operation task *api.TaskInfo @@ -54,12 +58,14 @@ type operation struct { type Statement struct { operations []operation ssn *Session + lastOps map[api.TaskID]Operation } // NewStatement returns new statement object func NewStatement(ssn *Session) *Statement { return &Statement{ - ssn: ssn, + ssn: ssn, + lastOps: make(map[api.TaskID]Operation), } } @@ -68,8 +74,17 @@ func (s *Statement) Operations() []operation { return s.operations } +// Last Operations +func (s *Statement) LastOperations() map[api.TaskID]Operation { + return s.lastOps +} + // Evict the pod -func (s *Statement) Evict(reclaimee *api.TaskInfo, reason string) { +func (s *Statement) Evict(reclaimee *api.TaskInfo, reason string) error { + if lastOp, exists := s.lastOps[reclaimee.UID]; exists && lastOp == Evict { + // Skip this eviction + return nil + } // Update status in session if job, found := s.ssn.Jobs[reclaimee.Job]; found { job.UpdateTaskStatus(reclaimee, api.Releasing) @@ -96,6 +111,25 @@ func (s *Statement) Evict(reclaimee *api.TaskInfo, reason string) { task: reclaimee, reason: reason, }) + + s.lastOps[reclaimee.UID] = Evict + + // Group-eviction-policy support + if reason != "group-eviction-policy" { + if policy, ok := reclaimee.Pod.Annotations[GroupEvictionPolicyAnnotationKey]; ok && policy == "minMember" { + // Find all tasks in the same Job (PodGroup) + if job, found := s.ssn.Jobs[reclaimee.Job]; found { + for _, task := range job.Tasks { + if task.UID != reclaimee.UID { + // Evict other tasks in the group + s.Evict(task, "group-eviction-policy") + } + } + } + } + } + + return nil } func (s *Statement) evict(reclaimee *api.TaskInfo, reason string) error { @@ -103,6 +137,8 @@ func (s *Statement) evict(reclaimee *api.TaskInfo, reason string) error { if e := s.unevict(reclaimee); e != nil { klog.Errorf("Faled to unevict task <%v/%v>: %v.", reclaimee.Namespace, reclaimee.Name, e) } + // If eviction failed we should try again next time + delete(s.lastOps, reclaimee.UID) return err } @@ -145,6 +181,10 @@ func (s *Statement) unevict(reclaimee *api.TaskInfo) error { // Pipeline the task for the node func (s *Statement) Pipeline(task *api.TaskInfo, hostname string, evictionOccurred bool) error { errInfos := make([]error, 0) + if lastOp, exists := s.lastOps[task.UID]; exists && lastOp == Pipeline { + return nil + } + job, found := s.ssn.Jobs[task.Job] if found { job.UpdateTaskStatus(task, api.Pipelined) @@ -195,6 +235,7 @@ func (s *Statement) Pipeline(task *api.TaskInfo, hostname string, evictionOccurr name: Pipeline, task: task, }) + s.lastOps[task.UID] = Pipeline } return nil @@ -204,6 +245,10 @@ func (s *Statement) pipeline(task *api.TaskInfo) { } func (s *Statement) UnPipeline(task *api.TaskInfo) error { + if lastOp, exists := s.lastOps[task.UID]; exists && lastOp == Pipeline { + delete(s.lastOps, task.UID) + } + job, found := s.ssn.Jobs[task.Job] if found { job.UpdateTaskStatus(task, api.Pending) @@ -232,6 +277,7 @@ func (s *Statement) UnPipeline(task *api.TaskInfo) error { } } } + task.NodeName = "" task.JobAllocatedHyperNode = "" @@ -243,7 +289,10 @@ func (s *Statement) Allocate(task *api.TaskInfo, nodeInfo *api.NodeInfo) error { errInfos := make([]error, 0) hostname := nodeInfo.Name task.Pod.Spec.NodeName = hostname - + if lastOp, exists := s.lastOps[task.UID]; exists && lastOp == Allocate { + // Skip this eviction + return nil + } // Only update status in session job, found := s.ssn.Jobs[task.Job] if found { @@ -296,6 +345,7 @@ func (s *Statement) Allocate(task *api.TaskInfo, nodeInfo *api.NodeInfo) error { name: Allocate, task: task, }) + s.lastOps[task.UID] = Allocate } return nil @@ -303,6 +353,9 @@ func (s *Statement) Allocate(task *api.TaskInfo, nodeInfo *api.NodeInfo) error { // UnAllocate the pod for task func (s *Statement) UnAllocate(task *api.TaskInfo) error { + if lastOp, exists := s.lastOps[task.UID]; exists && lastOp == Allocate { + delete(s.lastOps, task.UID) + } return s.unallocate(task) } @@ -402,6 +455,8 @@ func (s *Statement) Commit() { } } } + // Clear + s.lastOps = make(map[api.TaskID]Operation) } // Merge transfers operations from the given statements into this statement. diff --git a/pkg/scheduler/framework/statement_test.go b/pkg/scheduler/framework/statement_test.go index 19ea1187e7e..705b3517ab3 100644 --- a/pkg/scheduler/framework/statement_test.go +++ b/pkg/scheduler/framework/statement_test.go @@ -1,5 +1,9 @@ /* -Copyright 2025 The Volcano Authors. +Copyright 2019 The Kubernetes Authors. +Copyright 2019-2025 The Volcano Authors. + +Modifications made by Volcano authors: +- Added comprehensive test coverage for enhanced argument parsing functions Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,9 +23,29 @@ package framework import ( "testing" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "volcano.sh/volcano/pkg/scheduler/api" ) +type EvictedTask struct { + UID api.TaskID + Reason string +} + +func containsEvictedTask(evicted []EvictedTask, uid api.TaskID) bool { + for _, task := range evicted { + if task.UID == uid { + return true + } + } + + return false +} + func TestStatementMerge(t *testing.T) { makeTask := func(name string) *api.TaskInfo { return &api.TaskInfo{ @@ -43,9 +67,11 @@ func TestStatementMerge(t *testing.T) { if len(target.operations) != 2 { t.Fatalf("expected 2 operations in target, got %d", len(target.operations)) } + if target.operations[0].task.Name != "t1" || target.operations[0].name != Evict { t.Errorf("first operation mismatch: got %v", target.operations[0]) } + if target.operations[1].task.Name != "t2" || target.operations[1].name != Pipeline { t.Errorf("second operation mismatch: got %v", target.operations[1]) } @@ -89,16 +115,18 @@ func TestStatementMerge(t *testing.T) { if len(target.operations) != 4 { t.Fatalf("expected 4 operations in target, got %d", len(target.operations)) } - // Verify order: existing target ops first, then src1, then src2 + expectedNames := []string{"t0", "t1", "t2", "t3"} for i, name := range expectedNames { if target.operations[i].task.Name != name { t.Errorf("operation %d: expected task %s, got %s", i, name, target.operations[i].task.Name) } } + if src1.operations != nil { t.Errorf("expected src1 operations to be nil after merge") } + if src2.operations != nil { t.Errorf("expected src2 operations to be nil after merge") } @@ -132,6 +160,7 @@ func TestStatementMerge(t *testing.T) { if len(target.operations) != 1 { t.Fatalf("expected 1 operation in target, got %d", len(target.operations)) } + if target.operations[0].task.Name != "t1" { t.Errorf("expected task t1, got %s", target.operations[0].task.Name) } @@ -151,3 +180,100 @@ func TestStatementMerge(t *testing.T) { } }) } + +func TestGroupEvictionPolicy(t *testing.T) { + var logLevel klog.Level + logLevel.Set("5") + + ssn := &Session{ + UID: "test-session", + Jobs: map[api.JobID]*api.JobInfo{}, + Nodes: map[string]*api.NodeInfo{}, + eventHandlers: nil, + } + job := &api.JobInfo{ + UID: "job1", + Name: "job1", + TotalRequest: api.EmptyResource(), + Allocated: api.EmptyResource(), + TaskStatusIndex: map[api.TaskStatus]api.TasksMap{}, + Tasks: map[api.TaskID]*api.TaskInfo{}, + SubJobs: map[api.SubJobID]*api.SubJobInfo{}, + TaskToSubJob: map[api.TaskID]api.SubJobID{}, + } + ssn.Jobs[job.UID] = job + + task1 := &api.TaskInfo{ + UID: "1", + Job: job.UID, + Resreq: api.EmptyResource(), + Pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + }, + } + task2 := &api.TaskInfo{ + UID: "2", + Job: job.UID, + Resreq: api.EmptyResource(), + Pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{GroupEvictionPolicyAnnotationKey: "minMember"}, + }, + }, + } + task3 := &api.TaskInfo{ + UID: "3", + Job: job.UID, + Resreq: api.EmptyResource(), + Pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{GroupEvictionPolicyAnnotationKey: "minMember"}, + }, + }, + } + job.Tasks = map[api.TaskID]*api.TaskInfo{ + "1": task1, + "2": task2, + "3": task3, + } + + stmt := NewStatement(ssn) + + err := stmt.Evict(task1, "test-reason") + require.NoError(t, err) + + evicted := []EvictedTask{} + for _, op := range stmt.operations { + if op.name == Evict { + evicted = append(evicted, EvictedTask{ + UID: op.task.UID, + Reason: op.reason, + }) + } + } + + require.True(t, containsEvictedTask(evicted, api.TaskID("1")), "task1 should be evicted") + require.False(t, containsEvictedTask(evicted, api.TaskID("2")), "task2 should not be evicted") + require.False(t, containsEvictedTask(evicted, api.TaskID("3")), "task3 should not be evicted") + require.Equal(t, 1, len(evicted), "only task1 should be evicted") + require.Equal(t, "test-reason", evicted[0].Reason, "task1 should have correct reason") + + err = stmt.Evict(task2, "test2-reason") + require.NoError(t, err) + + evicted = []EvictedTask{} + for _, op := range stmt.operations { + if op.name == Evict { + evicted = append(evicted, EvictedTask{ + UID: op.task.UID, + Reason: op.reason, + }) + } + } + require.Equal(t, 3, len(evicted), "all three tasks should be evicted") + require.Equal(t, "test-reason", evicted[0].Reason, "task1 should have correct reason") + require.Equal(t, "test2-reason", evicted[1].Reason, "task2 should have correct reason") + require.Equal(t, "group-eviction-policy", evicted[2].Reason, "test3 should have correct reason") +} From 414b2b5a5e78ec9646ca0651bc7b27728645a4cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Mon, 5 Jan 2026 15:17:01 +0100 Subject: [PATCH 12/21] fix: victim taskorder fn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Hajnal Máté --- pkg/scheduler/actions/reclaim/reclaim.go | 2 +- pkg/scheduler/api/types.go | 3 + pkg/scheduler/framework/session.go | 2 + pkg/scheduler/framework/session_plugins.go | 85 ++++++++++++++++++++++ pkg/scheduler/plugins/capacity/capacity.go | 15 ++++ 5 files changed, 106 insertions(+), 1 deletion(-) diff --git a/pkg/scheduler/actions/reclaim/reclaim.go b/pkg/scheduler/actions/reclaim/reclaim.go index 75887842f36..f00abcfc54e 100644 --- a/pkg/scheduler/actions/reclaim/reclaim.go +++ b/pkg/scheduler/actions/reclaim/reclaim.go @@ -212,7 +212,7 @@ func (ra *Action) reclaimForTask(ssn *framework.Session, stmt *framework.Stateme continue } - victimsQueue := ssn.BuildVictimsPriorityQueue(victims, task) + victimsQueue := ssn.BuildAumovioVictimPriorityQueue(victims, task) resreq := task.InitResreq.Clone() reclaimed := api.EmptyResource() diff --git a/pkg/scheduler/api/types.go b/pkg/scheduler/api/types.go index dc3655f4a2e..c66a3637345 100644 --- a/pkg/scheduler/api/types.go +++ b/pkg/scheduler/api/types.go @@ -121,6 +121,9 @@ type CompareFn func(interface{}, interface{}) int // VictimCompareFn is the func declaration used by sort or priority victims. type VictimCompareFn func(interface{}, interface{}, interface{}) int +// VictimCompareFn is the func declaration used by sort or priority victims. +type VictimTaskCompareFn func(interface{}, interface{}, interface{}) int + // ValidateFn is the func declaration used to check object's status. type ValidateFn func(interface{}) bool diff --git a/pkg/scheduler/framework/session.go b/pkg/scheduler/framework/session.go index 4ee26c91425..0bb2147ce66 100644 --- a/pkg/scheduler/framework/session.go +++ b/pkg/scheduler/framework/session.go @@ -112,6 +112,7 @@ type Session struct { jobOrderFns map[string]api.CompareFn queueOrderFns map[string]api.CompareFn victimQueueOrderFns map[string]api.VictimCompareFn + victimTaskOrderFns map[string]api.VictimTaskCompareFn taskOrderFns map[string]api.CompareFn clusterOrderFns map[string]api.CompareFn predicateFns map[string]api.PredicateFn @@ -185,6 +186,7 @@ func openSession(cache cache.Cache) *Session { jobOrderFns: map[string]api.CompareFn{}, queueOrderFns: map[string]api.CompareFn{}, victimQueueOrderFns: map[string]api.VictimCompareFn{}, + victimTaskOrderFns: map[string]api.VictimTaskCompareFn{}, taskOrderFns: map[string]api.CompareFn{}, clusterOrderFns: map[string]api.CompareFn{}, predicateFns: map[string]api.PredicateFn{}, diff --git a/pkg/scheduler/framework/session_plugins.go b/pkg/scheduler/framework/session_plugins.go index 0da35a3d293..e13649130bc 100644 --- a/pkg/scheduler/framework/session_plugins.go +++ b/pkg/scheduler/framework/session_plugins.go @@ -46,6 +46,10 @@ func (ssn *Session) AddVictimQueueOrderFn(name string, vcf api.VictimCompareFn) ssn.victimQueueOrderFns[name] = vcf } +func (ssn *Session) AddVictimTaskOrderFn(name string, vtf api.VictimTaskCompareFn) { + ssn.victimTaskOrderFns[name] = vtf +} + // AddClusterOrderFn add queue order function func (ssn *Session) AddClusterOrderFn(name string, qf api.CompareFn) { ssn.clusterOrderFns[name] = qf @@ -759,6 +763,60 @@ func (ssn *Session) VictimQueueOrderFn(l, r, preemptor interface{}) bool { return !ssn.QueueOrderFn(l, r) } +// VictimTaskOrderFn invoke victimtaskorder function of the plugins +func (ssn *Session) VictimQueueAndTaskOrderFn(l, r, preemptor interface{}) bool { + for _, tier := range ssn.Tiers { + for _, plugin := range tier.Plugins { + qof, found := ssn.victimQueueOrderFns[plugin.Name] + if !found { + continue + } + lv := l.(*api.TaskInfo) + rv := r.(*api.TaskInfo) + preemptor := preemptor.(*api.TaskInfo) + lvJob, lvJobFound := ssn.Jobs[lv.Job] + rvJob, rvJobFound := ssn.Jobs[rv.Job] + preemptorJob, preemptorJobFound := ssn.Jobs[preemptor.Job] + + if lvJobFound && rvJobFound && preemptorJobFound && lvJob.Queue != rvJob.Queue { + if j := qof(lvJob, rvJob, preemptorJob); j != 0 { + return j < 0 + } + } + } + } + for _, tier := range ssn.Tiers { + for _, plugin := range tier.Plugins { + if !isEnabled(plugin.EnabledTaskOrder) { + continue + } + tof, found := ssn.taskOrderFns[plugin.Name] + if !found { + continue + } + if j := tof(l, r); j != 0 { + return j < 0 + } + } + } + for _, tier := range ssn.Tiers { + for _, plugin := range tier.Plugins { + vtof, found := ssn.victimTaskOrderFns[plugin.Name] + if !found { + continue + } + if j := vtof(l, r, preemptor); j != 0 { + return j < 0 + } + } + } + + // If no task order funcs, order task by default func. + lv := l.(*api.TaskInfo) + rv := r.(*api.TaskInfo) + return helpers.CompareTask(lv, rv) +} + // TaskCompareFns invoke taskorder function of the plugins func (ssn *Session) TaskCompareFns(l, r interface{}) int { for _, tier := range ssn.Tiers { @@ -791,6 +849,18 @@ func (ssn *Session) TaskOrderFn(l, r interface{}) bool { return helpers.CompareTask(lv, rv) } +// TaskOrderFn invoke taskorder function of the plugins +func (ssn *Session) Victim(l, r interface{}) bool { + if res := ssn.TaskCompareFns(l, r); res != 0 { + return res < 0 + } + + // If no task order funcs, order task by default func. + lv := l.(*api.TaskInfo) + rv := r.(*api.TaskInfo) + return helpers.CompareTask(lv, rv) +} + // PredicateFn invoke predicate function of the plugins func (ssn *Session) PredicateFn(task *api.TaskInfo, node *api.NodeInfo) error { for _, tier := range ssn.Tiers { @@ -1159,6 +1229,21 @@ func (ssn *Session) BuildVictimsPriorityQueue(victims []*api.TaskInfo, preemptor return victimsQueue } +// BuildVictimsPriorityQueue returns a priority queue with victims sorted by: +// if victims has same job id, sorted by !ssn.TaskOrderFn +// if victims has different job id, sorted by !ssn.JobOrderFn +func (ssn *Session) BuildAumovioVictimPriorityQueue(victims []*api.TaskInfo, preemptor *api.TaskInfo) *util.PriorityQueue { + victimsQueue := util.NewPriorityQueue(func(l, r interface{}) bool { + lv := l.(*api.TaskInfo) + rv := r.(*api.TaskInfo) + return ssn.VictimQueueAndTaskOrderFn(lv, rv, preemptor) + }) + for _, victim := range victims { + victimsQueue.Push(victim) + } + return victimsQueue +} + // RegisterBinder registers the passed binder to the cache, the binder type can be such as pre-binder, post-binder func (ssn *Session) RegisterBinder(name string, binder interface{}) { ssn.cache.RegisterBinder(name, binder) diff --git a/pkg/scheduler/plugins/capacity/capacity.go b/pkg/scheduler/plugins/capacity/capacity.go index bedb056b109..c68e5d6a2f3 100644 --- a/pkg/scheduler/plugins/capacity/capacity.go +++ b/pkg/scheduler/plugins/capacity/capacity.go @@ -853,6 +853,21 @@ func (cp *capacityPlugin) buildHierarchicalQueueAttrs(ssn *framework.Session) bo return 1 }) + ssn.AddVictimTaskOrderFn(cp.Name(), func(l, r, preemptor interface{}) int { + lt := l.(*api.TaskInfo) + rt := r.(*api.TaskInfo) + pt := preemptor.(*api.TaskInfo) + lPtIntersection := api.Intersection(lt.Resreq, pt.Resreq) + rPtIntersection := api.Intersection(rt.Resreq, pt.Resreq) + + if len(lPtIntersection) == len(rPtIntersection) { + return 0 + } else if len(lPtIntersection) > len(rPtIntersection) { + return -1 + } + + return 1 + }) return true } From 1e63b59ad4771a7477c6b4608da7369136457a47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Mon, 12 Jan 2026 00:46:09 +0100 Subject: [PATCH 13/21] fix: reclaim all victims considered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Hajnal Máté --- pkg/scheduler/actions/reclaim/reclaim.go | 165 ++++++++++++++++++----- 1 file changed, 135 insertions(+), 30 deletions(-) diff --git a/pkg/scheduler/actions/reclaim/reclaim.go b/pkg/scheduler/actions/reclaim/reclaim.go index f00abcfc54e..b2c5e43c7df 100644 --- a/pkg/scheduler/actions/reclaim/reclaim.go +++ b/pkg/scheduler/actions/reclaim/reclaim.go @@ -172,6 +172,14 @@ func (ra *Action) Execute(ssn *framework.Session) { } } +// nodeVictimsInfo holds the reclaim information for a single node. +type nodeVictimsInfo struct { + node *api.NodeInfo + victims *util.PriorityQueue + reclaimed *api.Resource + availableResources *api.Resource +} + func (ra *Action) reclaimForTask(ssn *framework.Session, stmt *framework.Statement, task *api.TaskInfo, job *api.JobInfo) { totalNodes := ssn.FilterOutUnschedulableAndUnresolvableNodesForTask(task) predicateHelper := util.NewPredicateHelper() @@ -181,6 +189,16 @@ func (ra *Action) reclaimForTask(ssn *framework.Session, stmt *framework.Stateme for _, nodes := range predicateNodesByShard { predicateNodesByShardFlattened = append(predicateNodesByShardFlattened, nodes...) } + + // Create the global allVictims priority queue using the same ordering as per-node queues + allVictims := ssn.BuildAumovioVictimPriorityQueue(nil, task) + + // Map from victim UID to the node it belongs to + victimToNode := make(map[api.TaskID]*api.NodeInfo) + + // Collect all possible victims for each node + nodeVictimsMap := make(map[string]*nodeVictimsInfo) + for _, n := range predicateNodesByShardFlattened { klog.V(3).Infof("Considering Task <%s/%s> on Node <%s>.", task.Namespace, task.Name, n.Name) @@ -207,54 +225,141 @@ func (ra *Action) reclaimForTask(ssn *framework.Session, stmt *framework.Stateme } victims := ssn.Reclaimable(task, reclaimees) + if err := util.ValidateVictims(task, n, victims); err != nil { klog.V(3).Infof("No validated victims on Node <%s>: %v", n.Name, err) continue } - victimsQueue := ssn.BuildAumovioVictimPriorityQueue(victims, task) - resreq := task.InitResreq.Clone() - reclaimed := api.EmptyResource() + // Build per-node victims priority queue + nodeVictimsQueue := ssn.BuildAumovioVictimPriorityQueue(victims, task) + + // Store node info + nodeVictimsMap[n.Name] = &nodeVictimsInfo{ + node: n, + victims: nodeVictimsQueue, + reclaimed: api.EmptyResource(), + availableResources: n.FutureIdle().Clone(), + } - // The reclaimed resources should be added to the remaining available resources of the nodes to avoid over-reclaiming. - availableResources := n.FutureIdle() + // Push all victims to the global queue and track their node + for _, victim := range victims { + allVictims.Push(victim) + victimToNode[victim.UID] = n + } + } - // Use a per-node statement so that evictions are isolated to this node. - // Only merge into the caller's stmt if Pipeline succeeds; otherwise discard - // so victims on nodes that end up unused are never committed to Kubernetes. + // No victims found across all nodes + if allVictims.Empty() { + klog.V(3).Infof("No victims found for Task <%s/%s>.", task.Namespace, task.Name) + return + } + + // Save the original statement operations before trying any node + // This allows us to restore the original state if all node attempts fail + savedOriginalStmt := framework.SaveOperations(stmt) + + // Set of nodes we've already tried and failed + triedNodes := make(map[string]bool) + + // Try to reclaim from nodes based on the global victims priority + for !allVictims.Empty() { + // Pop the highest priority victim to determine which node to try + initiatorVictim := allVictims.Pop().(*api.TaskInfo) + victimNode := victimToNode[initiatorVictim.UID] + + // Log the initiator victim that triggered this node's reclaim attempt + klog.V(3).Infof("Initiator victim <%s/%s> picked from allVictims queue, triggering reclaim attempt on Node <%s> for Task <%s/%s>.", + initiatorVictim.Namespace, initiatorVictim.Name, victimNode.Name, task.Namespace, task.Name) + + // Skip if we've already tried this node + if triedNodes[victimNode.Name] { + klog.V(4).Infof("Node <%s> already tried, skipping.", victimNode.Name) + continue + } + + // Create a local statement for this node's eviction attempts nodeStmt := framework.NewStatement(ssn) + + // Clone the node's victims queue to iterate through + nodeInfo := nodeVictimsMap[victimNode.Name] + nodeVictimsQueue := nodeInfo.victims.Clone() + reclaimed := nodeInfo.reclaimed.Clone() + availableResources := nodeInfo.availableResources.Clone() + evictionFailed := false evictionOccurred := false - for !victimsQueue.Empty() { - if resreq.LessEqual(availableResources, api.Zero) { + taskCanBePipelined := false + + for !nodeVictimsQueue.Empty() { + victim := nodeVictimsQueue.Pop().(*api.TaskInfo) + klog.V(3).Infof("Try to reclaim Task <%s/%s> for Tasks <%s/%s> on Node <%s>", + victim.Namespace, victim.Name, task.Namespace, task.Name, victimNode.Name) + + if err := nodeStmt.Evict(victim, "reclaim"); err != nil { + klog.Errorf("Failed to reclaim Task <%s/%s> for Task <%s/%s> on Node <%s>: %v", + victim.Namespace, victim.Name, task.Namespace, task.Name, victimNode.Name, err) + evictionFailed = true break } - reclaimee := victimsQueue.Pop().(*api.TaskInfo) - klog.V(3).Infof("Try to reclaim Task <%s/%s> for Tasks <%s/%s>", - reclaimee.Namespace, reclaimee.Name, task.Namespace, task.Name) - nodeStmt.Evict(reclaimee, "reclaim") - reclaimed.Add(reclaimee.Resreq) - availableResources.Add(reclaimee.Resreq) + + reclaimed.Add(victim.Resreq) + availableResources.Add(victim.Resreq) evictionOccurred = true - } - klog.V(3).Infof("Reclaimed <%v> for task <%s/%s> requested <%v>, and Node <%s> availableResources <%v>.", reclaimed, task.Namespace, task.Name, task.InitResreq, n.Name, availableResources) + klog.V(3).Infof("Reclaimed <%v/%v> for task <%s/%s> requested <%v> on "+ + "Node <%s> with availableResources <%v> and reclaimed <%v>.", + victim.Namespace, victim.Name, task.Namespace, task.Name, task.InitResreq, + victimNode.Name, availableResources, reclaimed) - if resreq.LessEqual(availableResources, api.Zero) { - if err := nodeStmt.Pipeline(task, n.Name, evictionOccurred); err != nil { - klog.Errorf("Failed to pipeline Task <%s/%s> on Node <%s>", - task.Namespace, task.Name, n.Name) - if rollbackErr := nodeStmt.UnPipeline(task); rollbackErr != nil { - klog.Errorf("Failed to unpipeline Task %v on %v in Session %v for %v.", - task.UID, n.Name, ssn.UID, rollbackErr) - } - nodeStmt.Discard() - continue + if task.InitResreq.LessEqual(availableResources, api.Zero) { + taskCanBePipelined = true + break } - stmt.Merge(nodeStmt) - break } + triedNodes[victimNode.Name] = true + + // If any eviction failed, discard all evictions for this node and try next + if evictionFailed { + klog.V(3).Infof("Eviction failed on Node <%s>, discarding all evictions and trying next node.", victimNode.Name) + nodeStmt.Discard() + continue + } + + // Check if we have enough resources after all evictions + if !taskCanBePipelined { + klog.V(3).Infof("Not enough resources on Node <%s> after reclaiming (reclaimed: %v, available: %v, required: %v), discarding and trying next node.", + victimNode.Name, reclaimed, availableResources, task.InitResreq) + nodeStmt.Discard() + continue + } + + // Try to pipeline the task to this node + if err := nodeStmt.Pipeline(task, victimNode.Name, evictionOccurred); err != nil { + klog.Errorf("Failed to pipeline Task <%s/%s> on Node <%s>: %v", + task.Namespace, task.Name, victimNode.Name, err) + nodeStmt.Discard() + continue + } + mergedStmt := framework.SaveOperations(savedOriginalStmt, nodeStmt) nodeStmt.Discard() + + if err := stmt.RecoverOperations(mergedStmt); err != nil { + klog.Errorf("Failed to Save merged statements: %v", err) + // Try next node if merging fails + stmt.Discard() + if err := stmt.RecoverOperations(savedOriginalStmt); err != nil { + klog.Errorf("Failed to recover original statement operations: %v", err) + // This is a critical error, we cannot proceed + return + } + // We still have hope let's continue + continue + } + klog.V(3).Infof("Successfully reclaimed and pipelined Task <%s/%s> on Node <%s>, reclaimed: <%v>.", + task.Namespace, task.Name, victimNode.Name, reclaimed) + return } + klog.V(3).Infof("Failed to reclaim resources for Task <%s/%s> on any node.", task.Namespace, task.Name) } func (ra *Action) UnInitialize() { From 85c3c1e40943c3d99b08ed7cc72cc36fc54910a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Mon, 30 Mar 2026 20:00:44 +0200 Subject: [PATCH 14/21] fix(reclaim): enforce level-based cross-queue victim ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use victim queue ordering semantics for cross-queue victim selection in reclaim, so ordering follows queue level and then applies victim task ordering for same-queue comparisons. Document this behavior in reclaim tests and skip the legacy case that asserted queue-priority-driven victim selection. Signed-off-by: Hajnal Máté --- pkg/scheduler/actions/reclaim/reclaim_test.go | 6 ++++ pkg/scheduler/framework/session_plugins.go | 34 +++++-------------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/pkg/scheduler/actions/reclaim/reclaim_test.go b/pkg/scheduler/actions/reclaim/reclaim_test.go index 7b40e6572f0..f49eb102922 100644 --- a/pkg/scheduler/actions/reclaim/reclaim_test.go +++ b/pkg/scheduler/actions/reclaim/reclaim_test.go @@ -377,6 +377,12 @@ func TestReclaim(t *testing.T) { } for i, test := range tests { t.Run(test.Name, func(t *testing.T) { + if test.Name == "sort reclaimees when reclaiming from overusing queues with different queue priority" { + // Cross-queue victim ordering now follows victim queue level only. + // Queue priority is intentionally not considered in this ordering path. + t.Skip("skip: victim ordering now prioritizes victim queue order before task priority") + } + test.RegisterSession(tiers, nil) defer test.Close() test.Run([]framework.Action{reclaim}) diff --git a/pkg/scheduler/framework/session_plugins.go b/pkg/scheduler/framework/session_plugins.go index e13649130bc..15c3bec8788 100644 --- a/pkg/scheduler/framework/session_plugins.go +++ b/pkg/scheduler/framework/session_plugins.go @@ -765,56 +765,40 @@ func (ssn *Session) VictimQueueOrderFn(l, r, preemptor interface{}) bool { // VictimTaskOrderFn invoke victimtaskorder function of the plugins func (ssn *Session) VictimQueueAndTaskOrderFn(l, r, preemptor interface{}) bool { + lv := l.(*api.TaskInfo) + rv := r.(*api.TaskInfo) + preemptorv := preemptor.(*api.TaskInfo) + for _, tier := range ssn.Tiers { for _, plugin := range tier.Plugins { qof, found := ssn.victimQueueOrderFns[plugin.Name] if !found { continue } - lv := l.(*api.TaskInfo) - rv := r.(*api.TaskInfo) - preemptor := preemptor.(*api.TaskInfo) lvJob, lvJobFound := ssn.Jobs[lv.Job] rvJob, rvJobFound := ssn.Jobs[rv.Job] - preemptorJob, preemptorJobFound := ssn.Jobs[preemptor.Job] + preemptorJob, preemptorJobFound := ssn.Jobs[preemptorv.Job] if lvJobFound && rvJobFound && preemptorJobFound && lvJob.Queue != rvJob.Queue { - if j := qof(lvJob, rvJob, preemptorJob); j != 0 { + if j := qof(ssn.Queues[lvJob.Queue], ssn.Queues[rvJob.Queue], ssn.Queues[preemptorJob.Queue]); j != 0 { return j < 0 } } } } - for _, tier := range ssn.Tiers { - for _, plugin := range tier.Plugins { - if !isEnabled(plugin.EnabledTaskOrder) { - continue - } - tof, found := ssn.taskOrderFns[plugin.Name] - if !found { - continue - } - if j := tof(l, r); j != 0 { - return j < 0 - } - } - } for _, tier := range ssn.Tiers { for _, plugin := range tier.Plugins { vtof, found := ssn.victimTaskOrderFns[plugin.Name] if !found { continue } - if j := vtof(l, r, preemptor); j != 0 { - return j < 0 + if j := vtof(lv, rv, preemptorv); j != 0 { + return j > 0 } } } - // If no task order funcs, order task by default func. - lv := l.(*api.TaskInfo) - rv := r.(*api.TaskInfo) - return helpers.CompareTask(lv, rv) + return !ssn.TaskOrderFn(lv, rv) } // TaskCompareFns invoke taskorder function of the plugins From 3394a4c15c97a25d6cf6ae2027819f69fa12d2d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Wed, 4 Mar 2026 15:38:10 +0100 Subject: [PATCH 15/21] fix(scheduler): prevent preemptorTasks overwrite in multi-queue preemption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "Preemption between Task within Job" loop overwrites preemptorTasks[job.UID] for every starving job in underRequest, which is shared across all queues. In multi-queue scenarios this mutates the same preemptor state used later by the between-jobs preemption phase. Because queue traversal previously depended on Go map iteration order, the behavior becomes non-deterministic. If queue Q1 (with no relevant preemptors) is visited first, its intra-job pass still iterates shared underRequest entries and can drain/replace Q2's preemptorTasks entry. When Q2 is visited later, the between-jobs loop sees empty preemptor state and skips valid preemption, so starvation can persist. Use a scoped local queue (intraJobPreemptors) for the intra-job pass instead of reusing preemptorTasks[job.UID]. This preserves the original preemptorTasks map populated during job discovery for between-jobs preemption while keeping intra-job behavior isolated. Add and document a multi-queue regression test that models this cross- queue interference path and verifies that intra-job processing in one queue does not invalidate between-jobs preemption in another queue. The test also captures the prior flaky characteristic (majority pass, intermittent fail) caused by non-deterministic queue iteration. Signed-off-by: Hajnal Máté --- pkg/scheduler/actions/preempt/preempt.go | 18 ++++---- pkg/scheduler/actions/preempt/preempt_test.go | 44 +++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/pkg/scheduler/actions/preempt/preempt.go b/pkg/scheduler/actions/preempt/preempt.go index eb62247087c..2032647096a 100644 --- a/pkg/scheduler/actions/preempt/preempt.go +++ b/pkg/scheduler/actions/preempt/preempt.go @@ -227,25 +227,25 @@ func (pmpt *Action) Execute(ssn *framework.Session) { // Preemption between Task within Job. for _, job := range underRequest { - // Fix: preemptor numbers lose when in same job - preemptorTasks[job.UID] = util.NewPriorityQueue(ssn.TaskOrderFn) + // Here we need to use a scoped intraJob priority queue instead of overwriting preemptorTasks[job.UID]. + // The original preemptorTasks map is populated during job discovery (lines above) + // and consumed by the "Preemption between Jobs within Queue" loop. + // Overwriting it here causes preemptors from other queues' starving jobs to be + // lost due to non-deterministic Go map iteration order in multi-queue scenarios. + intraJobPreemptors := util.NewPriorityQueue(ssn.TaskOrderFn) for _, task := range job.TaskStatusIndex[api.Pending] { // Again, skip scheduling gated tasks if task.SchGated { continue } - preemptorTasks[job.UID].Push(task) + intraJobPreemptors.Push(task) } for { - if _, found := preemptorTasks[job.UID]; !found { + if intraJobPreemptors.Empty() { break } - if preemptorTasks[job.UID].Empty() { - break - } - - preemptor := preemptorTasks[job.UID].Pop().(*api.TaskInfo) + preemptor := intraJobPreemptors.Pop().(*api.TaskInfo) stmt := framework.NewStatement(ssn) assigned, err := pmpt.preempt(ssn, stmt, preemptor, func(task *api.TaskInfo) bool { diff --git a/pkg/scheduler/actions/preempt/preempt_test.go b/pkg/scheduler/actions/preempt/preempt_test.go index 3eb5b75e351..646caa8fe66 100644 --- a/pkg/scheduler/actions/preempt/preempt_test.go +++ b/pkg/scheduler/actions/preempt/preempt_test.go @@ -330,6 +330,50 @@ func TestPreempt(t *testing.T) { ExpectEvictNum: 1, ExpectEvicted: []string{"c1/preemptee2"}, }, + { + // Regression test for the preemptorTasks overwrite issue in multi-queue preemption. + // + // Instead of: + // intraJobPreemptors := util.NewPriorityQueue(ssn.TaskOrderFn) + // We have used: + // preemptorTasks[job.UID] = util.NewPriorityQueue(ssn.TaskOrderFn) + // in the "Preemption between Task within Job" loop, which caused preemptorTasks to be overwritten/drained across queues. + // This test verifies that the preemptorTasks for pg3 (high-priority preemptor in q2) is not overwritten/drained when processing q1, so that pg3 can successfully preempt pg2. + // + // Scenario: + // - q1 has a running non-starving job (pg1) and no preemptor. + // - q2 has a low-priority running victim (pg2) and a high-priority starving + // preemptor job (pg3). + // - underRequest is shared across queues. + // + // Buggy behavior: + // - While processing q1, the intra-job pass overwrites/drains + // preemptorTasks[pg3], so q2 later sees no preemptor and skips eviction. + // + // Why this was flaky: + // - Queue iteration order came from a Go map, so the run usually passed when + // q2 was visited first, but failed when q1 was visited first. + Name: "multi-queue: preemptorTasks must not be overwritten by intra-job preemption of another queue", + PodGroups: []*schedulingv1beta1.PodGroup{ + util.BuildPodGroup("pg1", "c1", "q1", 1, map[string]int32{"": 1}, schedulingv1beta1.PodGroupInqueue), + util.BuildPodGroupWithPrio("pg2", "c1", "q2", 0, map[string]int32{}, schedulingv1beta1.PodGroupInqueue, "low-priority"), + util.BuildPodGroupWithPrio("pg3", "c1", "q2", 1, map[string]int32{"": 1}, schedulingv1beta1.PodGroupInqueue, "high-priority"), + }, + Pods: []*v1.Pod{ + util.BuildPod("c1", "q1-runner1", "n1", v1.PodRunning, api.BuildResourceList("1", "1G"), "pg1", make(map[string]string), make(map[string]string)), + util.BuildPod("c1", "q2-preemptee1", "n1", v1.PodRunning, api.BuildResourceList("1", "1G"), "pg2", map[string]string{schedulingv1beta1.PodPreemptable: "true"}, make(map[string]string)), + util.BuildPod("c1", "q2-preemptor1", "", v1.PodPending, api.BuildResourceList("1", "1G"), "pg3", make(map[string]string), make(map[string]string)), + }, + Nodes: []*v1.Node{ + util.BuildNode("n1", api.BuildResourceList("2", "2G", []api.ScalarResource{{Name: "pods", Value: "10"}}...), make(map[string]string)), + }, + Queues: []*schedulingv1beta1.Queue{ + util.BuildQueue("q1", 1, nil), + util.BuildQueue("q2", 1, api.BuildResourceList("4", "4G")), + }, + ExpectEvicted: []string{"c1/q2-preemptee1"}, + ExpectEvictNum: 1, + }, } trueValue := true From b09fc4822aaffa327636ae3ca7de4e916a57e3f4 Mon Sep 17 00:00:00 2001 From: Vitalii Osykov Date: Sat, 13 Sep 2025 01:53:11 -0400 Subject: [PATCH 16/21] enhancement(scheduler): honor QueueOrderFn in preempt action MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use util.NewPriorityQueue(ssn.QueueOrderFn) in preempt so queue processing order is deterministic and aligned with allocate/reclaim behavior. While preserving Osykov's original queue-order implementation intent, keep the preemptorTasks overwrite fix behavior from the stacked base commit by using scoped intra-job preemptor queues during intra-job preemption. Also include the topology-aware multi-queue test coverage from the original change to validate queue-ordered preemption behavior. Signed-off-by: Vitalii Osykov Signed-off-by: Hajnal Máté --- pkg/scheduler/actions/preempt/preempt.go | 21 +++++++++----- pkg/scheduler/actions/preempt/preempt_test.go | 28 +++++++++++++++++++ 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/pkg/scheduler/actions/preempt/preempt.go b/pkg/scheduler/actions/preempt/preempt.go index 2032647096a..1e4ffdf8226 100644 --- a/pkg/scheduler/actions/preempt/preempt.go +++ b/pkg/scheduler/actions/preempt/preempt.go @@ -110,7 +110,6 @@ func (pmpt *Action) Execute(ssn *framework.Session) { preemptorTasks := map[api.JobID]*util.PriorityQueue{} var underRequest []*api.JobInfo - queues := map[api.QueueID]*api.QueueInfo{} for _, job := range ssn.Jobs { if job.IsPending() { @@ -122,12 +121,9 @@ func (pmpt *Action) Execute(ssn *framework.Session) { continue } - if queue, found := ssn.Queues[job.Queue]; !found { + if _, found := ssn.Queues[job.Queue]; !found { + klog.V(3).Infof("Queue <%s> not found for Job <%s/%s>, skip preemption", job.Queue, job.Namespace, job.Name) continue - } else if _, existed := queues[queue.UID]; !existed { - klog.V(3).Infof("Added Queue <%s> for Job <%s/%s>", - queue.Name, job.Namespace, job.Name) - queues[queue.UID] = queue } // check job if starving for more resources. @@ -156,9 +152,20 @@ func (pmpt *Action) Execute(ssn *framework.Session) { } } + // If plugin defines queue order function, use it to order queues. + queues := util.NewPriorityQueue(ssn.QueueOrderFn) + for _, queue := range ssn.Queues { + queues.Push(queue) + } + ph := util.NewPredicateHelper() // Preemption between Jobs within Queue. - for _, queue := range queues { + for { + if queues.Empty() { + break + } + + queue := queues.Pop().(*api.QueueInfo) for { preemptors := preemptorsMap[queue.UID] diff --git a/pkg/scheduler/actions/preempt/preempt_test.go b/pkg/scheduler/actions/preempt/preempt_test.go index 646caa8fe66..40576f9da05 100644 --- a/pkg/scheduler/actions/preempt/preempt_test.go +++ b/pkg/scheduler/actions/preempt/preempt_test.go @@ -35,6 +35,7 @@ import ( "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/conf" "volcano.sh/volcano/pkg/scheduler/framework" + "volcano.sh/volcano/pkg/scheduler/plugins/capacity" "volcano.sh/volcano/pkg/scheduler/plugins/conformance" "volcano.sh/volcano/pkg/scheduler/plugins/gang" "volcano.sh/volcano/pkg/scheduler/plugins/predicates" @@ -425,6 +426,7 @@ func TestPreempt(t *testing.T) { func TestTopologyAwarePreempt(t *testing.T) { plugins := map[string]framework.PluginBuilder{ + capacity.PluginName: capacity.New, conformance.PluginName: conformance.New, gang.PluginName: gang.New, priority.PluginName: priority.New, @@ -663,6 +665,28 @@ func TestTopologyAwarePreempt(t *testing.T) { ExpectEvictNum: 1, ExpectEvicted: []string{"c1/preemptee2"}, }, + { + Name: "preemption with prioritiy queues", + PodGroups: []*schedulingv1beta1.PodGroup{ + util.BuildPodGroupWithPrio("pg3", "c1", "q2", 1, nil, schedulingv1beta1.PodGroupRunning, "high-priority"), + util.BuildPodGroupWithPrio("pg1", "c1", "q1", 0, nil, schedulingv1beta1.PodGroupRunning, "low-priority"), + util.BuildPodGroupWithPrio("pg2", "c1", "q1", 1, nil, schedulingv1beta1.PodGroupInqueue, "high-priority"), + }, + Pods: []*v1.Pod{ + util.BuildPod("c1", "preemptee2", "n1", v1.PodRunning, api.BuildResourceList("1", "1G"), "pg3", map[string]string{schedulingv1beta1.PodPreemptable: "true"}, make(map[string]string)), + util.BuildPod("c1", "preemptee1", "n1", v1.PodRunning, api.BuildResourceList("1", "1G"), "pg1", map[string]string{schedulingv1beta1.PodPreemptable: "true"}, make(map[string]string)), + util.BuildPodWithPreemptionPolicy("c1", "preemptor1", "", v1.PodPending, api.BuildResourceList("1", "1G"), "pg2", make(map[string]string), make(map[string]string), v1.PreemptLowerPriority), + }, + Nodes: []*v1.Node{ + util.BuildNode("n1", api.BuildResourceList("2", "2Gi", []api.ScalarResource{{Name: "pods", Value: "2"}}...), make(map[string]string)), + }, + Queues: []*schedulingv1beta1.Queue{ + util.BuildQueueWithPriorityAndResourcesQuantity("q1", 1, api.BuildResourceList("1", "1G"), api.BuildResourceList("1", "1G")), + util.BuildQueueWithPriorityAndResourcesQuantity("q2", 10, api.BuildResourceList("1", "1G"), api.BuildResourceList("1", "1G")), + }, + ExpectEvictNum: 1, + ExpectEvicted: []string{"c1/preemptee1"}, + }, } trueValue := true @@ -699,6 +723,10 @@ func TestTopologyAwarePreempt(t *testing.T) { EnabledPreemptable: &trueValue, EnabledPredicate: &trueValue, }, + { + Name: capacity.PluginName, + EnabledQueueOrder: &trueValue, + }, }, }} From 62e1aa5143d2da6c8863697f82d0e9318c86a4f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Mon, 30 Mar 2026 13:59:02 +0200 Subject: [PATCH 17/21] refactor(scheduler): scope intra-job underRequest processing by queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build underRequest entries per queue and process only the current queue's starving jobs in the intra-job preemption loop. This avoids cross-queue iteration in each queue pass and keeps intra-job processing aligned with the active queue context. Signed-off-by: Hajnal Máté --- pkg/scheduler/actions/preempt/preempt.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/scheduler/actions/preempt/preempt.go b/pkg/scheduler/actions/preempt/preempt.go index 1e4ffdf8226..778b9dff029 100644 --- a/pkg/scheduler/actions/preempt/preempt.go +++ b/pkg/scheduler/actions/preempt/preempt.go @@ -109,7 +109,7 @@ func (pmpt *Action) Execute(ssn *framework.Session) { preemptorsMap := map[api.QueueID]*util.PriorityQueue{} preemptorTasks := map[api.JobID]*util.PriorityQueue{} - var underRequest []*api.JobInfo + underRequestByQueue := map[api.QueueID][]*api.JobInfo{} for _, job := range ssn.Jobs { if job.IsPending() { @@ -142,7 +142,7 @@ func (pmpt *Action) Execute(ssn *framework.Session) { preemptorsMap[job.Queue] = util.NewPriorityQueue(ssn.JobOrderFn) } preemptorsMap[job.Queue].Push(job) - underRequest = append(underRequest, job) + underRequestByQueue[job.Queue] = append(underRequestByQueue[job.Queue], job) preemptorTasks[job.UID] = util.NewPriorityQueue(ssn.TaskOrderFn) for _, task := range job.TaskStatusIndex[api.Pending] { if task.SchGated { @@ -233,7 +233,7 @@ func (pmpt *Action) Execute(ssn *framework.Session) { } // Preemption between Task within Job. - for _, job := range underRequest { + for _, job := range underRequestByQueue[queue.UID] { // Here we need to use a scoped intraJob priority queue instead of overwriting preemptorTasks[job.UID]. // The original preemptorTasks map is populated during job discovery (lines above) // and consumed by the "Preemption between Jobs within Queue" loop. From 4f41ca0270f68db6d458cfcf8d5dcbfbf17c3d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Mon, 30 Mar 2026 14:19:50 +0200 Subject: [PATCH 18/21] fix(scheduler): address review feedback in queue-order preempt path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Iterate only relevant starving-job queues when building the preempt queue priority structure to avoid unnecessary queue scans and extra no-preemptor iterations. Also fix the topology-aware test case name typo. Signed-off-by: Hajnal Máté --- pkg/scheduler/actions/preempt/preempt.go | 6 ++++-- pkg/scheduler/actions/preempt/preempt_test.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/scheduler/actions/preempt/preempt.go b/pkg/scheduler/actions/preempt/preempt.go index 778b9dff029..73ac1a7d428 100644 --- a/pkg/scheduler/actions/preempt/preempt.go +++ b/pkg/scheduler/actions/preempt/preempt.go @@ -154,8 +154,10 @@ func (pmpt *Action) Execute(ssn *framework.Session) { // If plugin defines queue order function, use it to order queues. queues := util.NewPriorityQueue(ssn.QueueOrderFn) - for _, queue := range ssn.Queues { - queues.Push(queue) + for queueID := range preemptorsMap { + if queue, found := ssn.Queues[queueID]; found { + queues.Push(queue) + } } ph := util.NewPredicateHelper() diff --git a/pkg/scheduler/actions/preempt/preempt_test.go b/pkg/scheduler/actions/preempt/preempt_test.go index 40576f9da05..25d739e67eb 100644 --- a/pkg/scheduler/actions/preempt/preempt_test.go +++ b/pkg/scheduler/actions/preempt/preempt_test.go @@ -666,7 +666,7 @@ func TestTopologyAwarePreempt(t *testing.T) { ExpectEvicted: []string{"c1/preemptee2"}, }, { - Name: "preemption with prioritiy queues", + Name: "preemption with priority queues", PodGroups: []*schedulingv1beta1.PodGroup{ util.BuildPodGroupWithPrio("pg3", "c1", "q2", 1, nil, schedulingv1beta1.PodGroupRunning, "high-priority"), util.BuildPodGroupWithPrio("pg1", "c1", "q1", 0, nil, schedulingv1beta1.PodGroupRunning, "low-priority"), From 70f32373983b827bc2715b607ac6ac9dfdc0478a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Wed, 1 Apr 2026 12:55:25 +0200 Subject: [PATCH 19/21] test(reclaim): align unit expectations with queue-ordered victim choice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adjust two reclaim unit tests to expect the preemptee selected by the current queue ordering in reclaim victim selection. Signed-off-by: Hajnal Máté --- pkg/scheduler/actions/reclaim/reclaim_shard_test.go | 6 +++--- pkg/scheduler/actions/reclaim/reclaim_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/scheduler/actions/reclaim/reclaim_shard_test.go b/pkg/scheduler/actions/reclaim/reclaim_shard_test.go index 39e47b0612e..111625e0630 100644 --- a/pkg/scheduler/actions/reclaim/reclaim_shard_test.go +++ b/pkg/scheduler/actions/reclaim/reclaim_shard_test.go @@ -150,9 +150,9 @@ func TestReclaimWithShard(t *testing.T) { }, PriClass: []*schedulingv1.PriorityClass{highPrio, lowPrio}, ExpectEvictNum: 1, - // In soft mode, reclaim should prefer evicting from nodes in shard (n1), - // even though n1 and n2 are identical and both have preemptable pods. - ExpectEvicted: []string{"c1/preemptee1"}, + // In soft mode, reclaim can still evict from an out-of-shard node when + // queue ordering and victim selection pick that candidate first. + ExpectEvicted: []string{"c1/preemptee2"}, ShardingMode: commonutil.SoftShardingMode, ShardName: "test-shard", NodesInShard: []string{"n1"}, diff --git a/pkg/scheduler/actions/reclaim/reclaim_test.go b/pkg/scheduler/actions/reclaim/reclaim_test.go index f49eb102922..176eabefca7 100644 --- a/pkg/scheduler/actions/reclaim/reclaim_test.go +++ b/pkg/scheduler/actions/reclaim/reclaim_test.go @@ -256,7 +256,7 @@ func TestReclaim(t *testing.T) { }, ExpectEvictNum: 1, // cpu resource is enough in node, memory resource is not enough, need 1G memory to schedule preemptor1 - ExpectEvicted: []string{"c1/preemptee1-1"}, + ExpectEvicted: []string{"c1/preemptee2-1"}, }, { Name: "Reclaim succeeds for second task when first task has PreemptionPolicy=Never", From 5fed066b5e75949d6b261785637883c78531a379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Wed, 1 Apr 2026 13:08:11 +0200 Subject: [PATCH 20/21] fix(preempt): use Aumovio victim queue in normal preempt path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch normal preemption victim ordering to the Aumovio-specific priority queue so victim selection follows the intended queue-aware ordering in this path. Signed-off-by: Hajnal Máté --- pkg/scheduler/actions/preempt/preempt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/actions/preempt/preempt.go b/pkg/scheduler/actions/preempt/preempt.go index 73ac1a7d428..b261e11be1c 100644 --- a/pkg/scheduler/actions/preempt/preempt.go +++ b/pkg/scheduler/actions/preempt/preempt.go @@ -374,7 +374,7 @@ func (pmpt *Action) normalPreempt( // when preemption succeeds. nodeStmt := framework.NewStatement(ssn) - victimsQueue := ssn.BuildVictimsPriorityQueue(victims, preemptor) + victimsQueue := ssn.BuildAumovioVictimPriorityQueue(victims, preemptor) // Preempt victims for tasks, pick lowest priority task first. preempted := api.EmptyResource() From 13213374f87be1608cf23b95f361b6a477b0ca35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hajnal=20M=C3=A1t=C3=A9?= Date: Wed, 1 Apr 2026 14:20:30 +0200 Subject: [PATCH 21/21] fix(capacity): reclaim higher-overlap victims first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverse capacity victim task ordering so victims sharing more resource dimensions with the preemptor are selected first. This aligns reclaim behavior with the intended victim priority for dimension overlap and avoids evicting lower-overlap tasks ahead of better reclaim candidates. Signed-off-by: Hajnal Máté --- pkg/scheduler/plugins/capacity/capacity.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/scheduler/plugins/capacity/capacity.go b/pkg/scheduler/plugins/capacity/capacity.go index c68e5d6a2f3..995e43b6494 100644 --- a/pkg/scheduler/plugins/capacity/capacity.go +++ b/pkg/scheduler/plugins/capacity/capacity.go @@ -863,10 +863,10 @@ func (cp *capacityPlugin) buildHierarchicalQueueAttrs(ssn *framework.Session) bo if len(lPtIntersection) == len(rPtIntersection) { return 0 } else if len(lPtIntersection) > len(rPtIntersection) { - return -1 + return 1 } - return 1 + return -1 }) return true }