diff --git a/.gitmodules b/.gitmodules index 634671b2..20c45e7c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "gvm-nvidia-driver-modules"] path = gvm-nvidia-driver-modules - url = git@github.com:ovg-project/gvm-nvidia-driver-modules.git + url = https://github.com/ovg-project/gvm-nvidia-driver-modules.git [submodule "gvm-cuda-driver"] path = gvm-cuda-driver - url = git@github.com:ovg-project/gvm-cuda-driver.git + url = https://github.com/ovg-project/gvm-cuda-driver.git diff --git a/README.md b/README.md index dddf9ea0..f04fae7e 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # GVM + GVM is an OS-level GPU virtualization layer which achieves hardware-like performance isolation while preserving the flexibility of software-based sharing GVM provides cgroup-like APIs for GPU applications so you can check and operate GPU applications like what you did on CPU applications. For details, please check [here](https://github.com/ovg-project/GVM/blob/main/assets/GVM_paper.pdf). | API | Description | -|:--------------------|:-------------------------------------------------------------------------------------------| +| :------------------ | :----------------------------------------------------------------------------------------- | | memory.limit | Check or set the maximum amount of memory that the application can allocate on GPU | | memory.current | Get the current memory usage of the application on GPU | | memory.swap.current | Get the current amount of memory swapped to host of the application on GPU | @@ -13,27 +14,33 @@ For details, please check [here](https://github.com/ovg-project/GVM/blob/main/as | gcgroup.stat | Get statistics about the application | ## Performance + The figure shows the performance benefits of GVM when colocating high priority task `vllm` and low priority task `diffusion` on A100-40G GPU. GVM can achieve **59x** better p99 TTFT in high priority task compared to second best baseline while still get the highert throughput on low priority task. Thanks to [@boyuan](https://github.com/boyuanjia1126) for decorating figure. ![](./assets/vllm+diffusion.png) # Requirements + 1. [GVM NVIDIA GPU Driver](https://github.com/ovg-project/gvm-nvidia-driver-modules) installed 2. [GVM CUDA Driver Intercept Layer](https://github.com/ovg-project/gvm-cuda-driver) installed 3. Dependencies: - 1. `python3` `python3-pip` `python3-venv` - 2. `gcc` `g++` `make` `cmake` - 3. `cuda-toolkit` `nvidia-open` + 1. `python3` `python3-pip` `python3-venv` + 2. `gcc` `g++` `make` `cmake` + 3. `cuda-toolkit` `nvidia-open` # Install applications + ``` ./setup {llama.cpp|diffusion|llamafactory|vllm|sglang} ``` # Example + ## diffuser + Launch your diffuser: + ``` source diffuser/bin/activate export LD_LIBRARY_PATH=:$LD_LIBRARY_PATH @@ -41,28 +48,34 @@ python3 diffuser/diffusion.py --dataset_path=diffuser/vidprom.txt --log_file=dif ``` Get pid of diffuser: + ``` export pid= ``` Check kernel submission stats: + ``` cat /sys/kernel/debug/nvidia-uvm/processes/$pid/0/gcgroup.stat ``` Check memory stats: + ``` cat /sys/kernel/debug/nvidia-uvm/processes/$pid/0/memory.current cat /sys/kernel/debug/nvidia-uvm/processes/$pid/0/memory.swap.current ``` Limit memory usage: + ``` echo | sudo tee /sys/kernel/debug/nvidia-uvm/processes/$pid/0/memory.limit ``` ## vllm + diffuser + Launch your vllm: + ``` source vllm/bin/activate export LD_LIBRARY_PATH=:$LD_LIBRARY_PATH @@ -70,6 +83,7 @@ vllm serve meta-llama/Llama-3.2-3B --gpu-memory-utilization 0.8 --disable-log-re ``` Launch your diffuser: + ``` source diffuser/bin/activate export LD_LIBRARY_PATH=:$LD_LIBRARY_PATH @@ -77,27 +91,32 @@ python3 diffuser/diffusion.py --dataset_path=diffuser/vidprom.txt --log_file=dif ``` Get pid of diffuser and vllm: + ``` export diffuserpid= export vllmpid= ``` Check compute priority of vllm: + ``` cat /sys/kernel/debug/nvidia-uvm/processes/$vllmpid/0/compute.priority ``` Set compute priority of vllm to 2 to use a larger timeslice: + ``` echo 2 | sudo tee /sys/kernel/debug/nvidia-uvm/processes/$vllmpid/0/compute.priority ``` Limit memory usage of diffuser to ~6GB to make enough room for vllm to run: + ``` echo 6000000000 | sudo tee /sys/kernel/debug/nvidia-uvm/processes/$diffuserpid/0/memory.limit ``` Generate workloads for vllm: + ``` source vllm/bin/activate vllm bench serve \ @@ -110,11 +129,172 @@ vllm bench serve \ ``` Preempt diffuser for even higher vllm performance: + ``` echo 1 | sudo tee /sys/kernel/debug/nvidia-uvm/processes/$diffuserpid/0/compute.freeze ``` After vllm workloads stop, reschedule diffuser: + ``` echo 0 | sudo tee /sys/kernel/debug/nvidia-uvm/processes/$diffuserpid/0/compute.freeze ``` + +# Troubleshooting + +This section documents common issues encountered during GVM setup and their solutions. + +## Issue 1: Git submodules fail to clone (SSH authentication) + +**Error:** + +``` +git@github.com: Permission denied (publickey). +fatal: Could not read from remote repository. +``` + +**Cause:** Submodule URLs use SSH (`git@github.com:...`) but SSH keys are not configured. + +**Fix:** Override submodule URLs to use HTTPS before initializing: + +```bash +git config submodule.gvm-cuda-driver.url https://github.com/ovg-project/gvm-cuda-driver.git +git config submodule.gvm-nvidia-driver-modules.url https://github.com/ovg-project/gvm-nvidia-driver-modules.git +git submodule update --init --recursive +``` + +--- + +## Issue 2: CUDA installation fails on kernel 6.8+ + +**Error:** + +``` +nvidia/nv-dmabuf.c:844:9: error: implicit declaration of function 'dma_buf_attachment_is_dynamic' +ERROR: The nvidia kernel module was not created. +``` + +**Cause:** The driver bundled with CUDA (575.57.08) is incompatible with kernels >= 6.8 due to removed kernel APIs. + +**Fix:** Install CUDA toolkit only (without bundled driver), then install GVM driver separately: + +```bash +# In gvm-nvidia-driver-modules/scripts/ +sudo sh cuda_12.9.1_575.57.08_linux.run --silent --toolkit --override --no-drm +sudo sh NVIDIA-Linux-x86_64-575.64.05.run --no-kernel-modules +``` + +When prompted for kernel module type, select **MIT/GPL (option 2)**. + +--- + +## Issue 3: Kernel module compilation fails on kernel 6.17+ + +**Error:** + +``` +nvidia-drm/nvidia-drm-fb.c:308:5: error: too few arguments to function 'drm_helper_mode_fill_fb_struct' +nvidia-drm/nvidia-drm-drv.c:240:18: error: initialization from incompatible pointer type +``` + +**Cause:** GVM kernel modules (based on NVIDIA 575.64.05) are incompatible with kernel 6.17+ due to DRM API changes. + +**Fix:** Downgrade to kernel 6.8 (tested with 6.8.0-1007-gcp on Ubuntu 24.04): + +```bash +sudo apt-get install -y linux-image-6.8.0-1007-gcp linux-headers-6.8.0-1007-gcp linux-modules-6.8.0-1007-gcp +``` + +On GCP cloud images, you may need to manually set the boot kernel in `/etc/default/grub.d/50-cloudimg-settings.cfg`: + +```bash +# Find the kernel entry ID +sudo grep -E "menuentry|submenu" /boot/grub/grub.cfg | head -20 + +# Update GRUB_DEFAULT (replace with actual UUID from above) +sudo sed -i "s|GRUB_DEFAULT=0|GRUB_DEFAULT='gnulinux-advanced->gnulinux-6.8.0-1007-gcp-advanced-'|" /etc/default/grub.d/50-cloudimg-settings.cfg + +sudo update-grub && sudo reboot +``` + +Verify after reboot: + +```bash +uname -r # should show 6.8.0-1007-gcp +``` + +--- + +## Issue 4: `nvcc` not found during CUDA intercept layer build + +**Error:** + +``` +/bin/sh: 1: nvcc: not found +make: *** [Makefile:30: build] Error 127 +``` + +**Cause:** CUDA toolkit installs `nvcc` to `/usr/local/cuda/bin/` which is not in `$PATH` by default. + +**Fix:** + +```bash +export PATH=/usr/local/cuda/bin:$PATH +``` + +Add this to your `~/.bashrc` to make it permanent: + +```bash +echo 'export PATH=/usr/local/cuda/bin:$PATH' >> ~/.bashrc +``` + +--- + +## Issue 5: HuggingFace gated model access denied + +**Error:** + +``` +huggingface_hub.errors.GatedRepoError: 401 Client Error. +Access to model stabilityai/stable-diffusion-3.5-medium is restricted. +``` + +**Cause:** Some models (like `stabilityai/stable-diffusion-3.5-medium`) require HuggingFace authentication and license acceptance. + +**Fix:** + +1. Accept the license at https://huggingface.co/stabilityai/stable-diffusion-3.5-medium +2. Generate a token at https://huggingface.co/settings/tokens (select "Read" access) +3. Authenticate on your system: + +```bash +python3 -c "from huggingface_hub import login; login(token='')" +``` + +--- + +## Issue 6: Diffusion log file fails to save + +**Error:** + +``` +FileNotFoundError: [Errno 2] No such file or directory: 'diffusion_outputs/diffusion/stats.txt' +``` + +**Cause:** The diffusion script writes logs to `diffusion_outputs/` but doesn't create the directory automatically. + +**Fix:** Create the directory before running: + +```bash +mkdir -p diffusion_outputs/diffusion +``` + +**Note:** Use `--num_requests=5` to run a quick smoke test (default is 10,000 requests which takes ~40 hours). + +--- + +## Additional Notes + +- **Tested environment:** GCP VM with Ubuntu 24.04.4 LTS, kernel 6.8.0-1007-gcp, NVIDIA L4 GPU +- **After reboot:** If GVM modules are not loaded, run `sudo ./deploy_modules.sh` from `~/GVM/gvm-nvidia-driver-modules/scripts/` +- **Before running GPU apps:** Always set `export LD_LIBRARY_PATH=~/GVM/gvm-cuda-driver/install:$LD_LIBRARY_PATH` diff --git a/SUBMODULE_FIXES.md b/SUBMODULE_FIXES.md new file mode 100644 index 00000000..87074b4e --- /dev/null +++ b/SUBMODULE_FIXES.md @@ -0,0 +1,111 @@ +# Required Fixes for Submodule Scripts + +This document outlines the changes needed in the submodule repositories (`gvm-nvidia-driver-modules` and `gvm-cuda-driver`) to address issues discovered during setup. + +## gvm-nvidia-driver-modules + +### Fix 1: Update `install_cuda.sh` to install toolkit only + +**File:** `scripts/install_cuda.sh` + +**Current code:** +```bash +#!/bin/bash +sudo sh cuda_12.9.1_575.57.08_linux.run --check +sudo sh cuda_12.9.1_575.57.08_linux.run --silent --driver --toolkit --override +``` + +**Issue:** The bundled driver (575.57.08) fails to compile on kernel 6.8+ due to removed kernel APIs. + +**Proposed fix:** +```bash +#!/bin/bash +sudo sh cuda_12.9.1_575.57.08_linux.run --check +# Install toolkit only, without bundled driver (which is incompatible with kernel 6.8+) +sudo sh cuda_12.9.1_575.57.08_linux.run --silent --toolkit --override --no-drm +echo "CUDA toolkit installed. Run install_nv_driver.sh next to install the GVM driver." +``` + +### Fix 2: Add kernel version check to `compile_modules.sh` + +**File:** `scripts/compile_modules.sh` + +**Proposed addition at the beginning:** +```bash +#!/bin/bash + +# Check kernel version compatibility +KERNEL_VERSION=$(uname -r | cut -d. -f1,2) +KERNEL_MAJOR=$(echo $KERNEL_VERSION | cut -d. -f1) +KERNEL_MINOR=$(echo $KERNEL_VERSION | cut -d. -f2) + +if [ "$KERNEL_MAJOR" -gt 6 ] || ([ "$KERNEL_MAJOR" -eq 6 ] && [ "$KERNEL_MINOR" -gt 8 ]); then + echo "WARNING: Kernel version $KERNEL_VERSION detected." + echo "GVM kernel modules are tested on kernel 6.8 and may not compile on newer kernels." + echo "If compilation fails, consider downgrading to kernel 6.8." + echo "" + read -p "Continue anyway? (y/n) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi +fi + +# Rest of the existing script... +``` + +### Fix 3: Add PATH setup reminder to README + +**File:** `README.md` + +Add a note after the CUDA installation step: +```markdown +**Note:** After installing CUDA, add it to your PATH: +```bash +export PATH=/usr/local/cuda/bin:$PATH +``` + +Add this to your `~/.bashrc` to make it permanent: +```bash +echo 'export PATH=/usr/local/cuda/bin:$PATH' >> ~/.bashrc +``` +``` + +## gvm-cuda-driver + +### Fix 1: Add PATH check to Makefile or build script + +**File:** `Makefile` or create a new `build.sh` wrapper + +**Proposed addition:** +```bash +#!/bin/bash +# build.sh - Wrapper script to ensure nvcc is in PATH + +if ! command -v nvcc &> /dev/null; then + echo "ERROR: nvcc not found in PATH" + echo "Please add CUDA to your PATH:" + echo " export PATH=/usr/local/cuda/bin:\$PATH" + echo "" + echo "Or add it permanently to ~/.bashrc:" + echo " echo 'export PATH=/usr/local/cuda/bin:\$PATH' >> ~/.bashrc" + exit 1 +fi + +# Run the actual make command +make "$@" +``` + +Then update README to suggest using `./build.sh` instead of `make` directly, or add the PATH check directly in the Makefile. + +## Summary of Changes + +These fixes address the following issues: +1. **Issue 2** - CUDA bundled driver incompatibility → Fixed by `install_cuda.sh` change +2. **Issue 3** - Kernel version incompatibility → Fixed by adding version check in `compile_modules.sh` +3. **Issue 4** - Missing nvcc in PATH → Fixed by adding PATH checks and documentation + +The other issues are already addressed in the main GVM repository: +- **Issue 1** - SSH URLs → Fixed by changing `.gitmodules` to HTTPS +- **Issue 5** - HuggingFace auth → Documented in README troubleshooting +- **Issue 6** - Diffusion output directory → Fixed by updating `setup` script diff --git a/gvm-docker/.gitignore b/gvm-docker/.gitignore new file mode 100644 index 00000000..d0d000d8 --- /dev/null +++ b/gvm-docker/.gitignore @@ -0,0 +1,3 @@ +# Binaries (root level only) +/gvm-daemon +/gvm-runc diff --git a/gvm-docker/API_DESIGN.md b/gvm-docker/API_DESIGN.md new file mode 100644 index 00000000..c2a663fa --- /dev/null +++ b/gvm-docker/API_DESIGN.md @@ -0,0 +1,970 @@ +# GVM API Design: Layered Architecture + +## Table of Contents + +1. [Architecture Overview](#1-architecture-overview) +2. [Layer 0: cgroup API (GVM Sysfs Interface)](#2-layer-0-cgroup-api) +3. [Layer 1: Docker](#3-layer-1-docker) +4. [Layer 2: Docker Compose](#4-layer-2-docker-compose) +5. [Layer 3: Kubernetes](#5-layer-3-kubernetes) +6. [Layer Integration: Top-Down Request Flow](#6-layer-integration) +7. [Layer Integration: Bottom-Up Visibility](#7-bottom-up-visibility) + +--- + +## 1. Architecture Overview + +GVM exposes GPU resource controls through a layered stack. Each layer provides a progressively higher-level abstraction while ultimately mapping down to the same kernel-level cgroup API. + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Layer 3: Kubernetes │ +│ Pod resource requests → Device Plugin → Container env vars │ +├─────────────────────────────────────────────────────────────┤ +│ Layer 2: Docker Compose │ +│ YAML service definitions → docker run commands │ +├─────────────────────────────────────────────────────────────┤ +│ Layer 1: Docker │ +│ Container env vars → GVM Daemon → sysfs writes │ +├─────────────────────────────────────────────────────────────┤ +│ Layer 0: cgroup API (GVM Sysfs Interface) │ +│ /sys/kernel/debug/nvidia-uvm/processes/// │ +│ memory.limit | memory.current | compute.priority | ... │ +└─────────────────────────────────────────────────────────────┘ +``` + +**Design Principle:** Each layer only talks to the layer directly below it. No layer skips levels. + +--- + +## 2. Layer 0: cgroup API + +This is the kernel-level interface provided by the GVM NVIDIA driver modules. All GPU resource control ultimately happens here. + +### 2.1 Sysfs Path Structure + +``` +/sys/kernel/debug/nvidia-uvm/processes/// +``` + +- **``** — Linux process ID of the GPU-using process +- **``** — GPU device index (0, 1, 2, ...) +- **``** — Control file (see table below) + +### 2.2 API Reference + +| File | R/W | Type | Description | +|:-----|:----|:-----|:------------| +| `memory.limit` | RW | uint64 (bytes) | Maximum GPU memory the process can allocate. Default: unlimited (`18446744073709551615`). Write a byte value to enforce a cap. | +| `memory.current` | R | uint64 (bytes) | Current GPU memory usage of the process. | +| `memory.swap.current` | R | uint64 (bytes) | Amount of GPU memory currently swapped to host RAM. | +| `compute.priority` | RW | int (0–15) | Compute scheduling priority. **0 = highest priority, 15 = lowest priority.** Default: 8. | +| `compute.freeze` | RW | bool (0 or 1) | Freeze (`1`) or unfreeze (`0`) all GPU compute for this process. Frozen processes yield all GPU time. | +| `gcgroup.stat` | R | text | Kernel submission statistics for the process. | + +### 2.3 Usage Examples + +```bash +# Read current memory usage +cat /sys/kernel/debug/nvidia-uvm/processes/12345/0/memory.current + +# Set memory limit to 6 GB +echo 6000000000 | sudo tee /sys/kernel/debug/nvidia-uvm/processes/12345/0/memory.limit + +# Set high priority (low number = high priority) +echo 2 | sudo tee /sys/kernel/debug/nvidia-uvm/processes/12345/0/compute.priority + +# Freeze a process +echo 1 | sudo tee /sys/kernel/debug/nvidia-uvm/processes/12345/0/compute.freeze + +# Unfreeze a process +echo 0 | sudo tee /sys/kernel/debug/nvidia-uvm/processes/12345/0/compute.freeze + +# Read swap usage +cat /sys/kernel/debug/nvidia-uvm/processes/12345/0/memory.swap.current + +# Read kernel submission stats +cat /sys/kernel/debug/nvidia-uvm/processes/12345/0/gcgroup.stat +``` + +### 2.4 Semantics + +- **memory.limit**: The kernel enforces this at the page-fault level. When a process exceeds its limit, allocations trigger memory swap to host or are denied. This cannot be bypassed from userspace. +- **compute.priority**: The kernel GPU scheduler uses this to determine timeslice allocation. A process with priority 0 gets the largest timeslice; priority 15 gets the smallest. Two colocated processes at different priorities will see proportionally different GPU time. +- **compute.freeze**: Immediately suspends all GPU kernel submissions for the process. The process remains alive but its GPU work is paused. Useful for preemption scenarios. +- **Per-GPU scoping**: Each control file is scoped to a specific GPU index. A multi-GPU process will have separate directories for each GPU (e.g., `/0/`, `/1/`). + +### 2.5 Constraints + +- Requires root or appropriate permissions to write to debugfs. +- PID directory only appears after the process makes its first GPU allocation. +- Controls are per-process, not per-container. A container with multiple GPU processes needs each one controlled separately. + +--- + +## 3. Layer 1: Docker + +The Docker layer translates **container-level intent** (environment variables) into **process-level sysfs writes** via the GVM Docker Daemon. + +### 3.1 Container Environment Variables + +Users declare GPU controls as environment variables when launching containers: + +| Environment Variable | Maps To | Format | Example | +|:---------------------|:--------|:-------|:--------| +| `GVM_MEMORY_LIMIT` | `memory.limit` | Human-readable size: `` where unit is `b`, `k`, `m`, `g`, `t` (case-insensitive). Plain integer = bytes. | `6g`, `6000m`, `6000000000` | +| `GVM_MEMORY_LIMIT_` | `memory.limit` on GPU N | Same as above, per-device override | `GVM_MEMORY_LIMIT_0=8g` | +| `GVM_MEMORY_PERCENTAGE` | `memory.limit` (computed) | Integer 1–100. Converted to bytes based on total GPU memory. | `50` | +| `GVM_COMPUTE_PRIORITY` | `compute.priority` | Integer 0–15. **0 = highest, 15 = lowest.** | `2` | +| `GVM_COMPUTE_FREEZE` | `compute.freeze` | `true` or `false` | `false` | +| `GVM_ENABLED` | *(control toggle)* | `true` or `false`. If `false`, daemon skips this container. Default: `true`. | `true` | +| `GVM_DEBUG` | *(logging toggle)* | `true` or `false`. Enables verbose daemon logging for this container. | `false` | + +**Unit parsing for `GVM_MEMORY_LIMIT`:** + +| Input | Bytes | +|:------|:------| +| `6g` or `6G` | 6,000,000,000 (6 × 10⁹) | +| `6000m` or `6000M` | 6,000,000,000 (6000 × 10⁶) | +| `1500k` or `1500K` | 1,500,000 (1500 × 10³) | +| `6000000000` | 6,000,000,000 (raw bytes) | + +> **Note:** We use SI units (powers of 10) not binary units (powers of 2). `1g` = 1,000,000,000 bytes, not 1,073,741,824. This is consistent with GPU memory reporting conventions. + +### 3.2 Docker Run Examples + +```bash +# Basic: 6GB memory limit, high priority +docker run -d --gpus all \ + -e GVM_MEMORY_LIMIT=6g \ + -e GVM_COMPUTE_PRIORITY=2 \ + my-gpu-app + +# Percentage-based memory limit (50% of GPU memory) +docker run -d --gpus all \ + -e GVM_MEMORY_PERCENTAGE=50 \ + -e GVM_COMPUTE_PRIORITY=8 \ + my-gpu-app + +# Per-device memory limits for multi-GPU +docker run -d --gpus all \ + -e GVM_MEMORY_LIMIT_0=8g \ + -e GVM_MEMORY_LIMIT_1=4g \ + -e GVM_COMPUTE_PRIORITY=5 \ + multi-gpu-app + +# Disable GVM for a specific container +docker run -d --gpus all \ + -e GVM_ENABLED=false \ + my-unmanaged-app + +# Debug mode +docker run -d --gpus all \ + -e GVM_MEMORY_LIMIT=6g \ + -e GVM_COMPUTE_PRIORITY=2 \ + -e GVM_DEBUG=true \ + my-gpu-app +``` + +### 3.3 Colocation Example + +```bash +# High-priority inference server (priority 2 = high) +docker run -d --gpus all \ + --name inference \ + -e GVM_MEMORY_LIMIT=16g \ + -e GVM_COMPUTE_PRIORITY=2 \ + vllm-server + +# Low-priority training job (priority 12 = low) +docker run -d --gpus all \ + --name training \ + -e GVM_MEMORY_LIMIT=8g \ + -e GVM_COMPUTE_PRIORITY=12 \ + training-image +``` + +### 3.4 GVM Docker Daemon + +The daemon is the **bridge** between Docker and the cgroup API. It runs on the host as a privileged systemd service. + +**Responsibilities:** +1. Poll running Docker containers (via `docker inspect`) +2. Read `GVM_*` environment variables from each container +3. Discover GPU processes belonging to each container (by matching container PID tree against `/sys/kernel/debug/nvidia-uvm/processes/`) +4. Parse and convert environment variable values into raw sysfs values +5. Write values to the appropriate sysfs files +6. Re-check periodically for new GPU processes (handles late GPU initialization) + +**Daemon lifecycle:** +``` + ┌──────────────┐ + │ Daemon Loop │ (every 5s) + └──────┬───────┘ + │ + ┌───────────▼───────────┐ + │ docker ps -q │ List running containers + └───────────┬───────────┘ + │ + ┌───────────▼───────────┐ + │ docker inspect │ Get PID + env vars + └───────────┬───────────┘ + │ + ┌────────▼────────┐ + │ Has GVM_* vars? │ + └──┬──────────┬───┘ + │ No │ Yes + │ (skip) │ + │ ┌─────▼──────────────┐ + │ │ Parse env vars │ + │ │ GVM_MEMORY_LIMIT │ → bytes + │ │ GVM_COMPUTE_PRIORITY│ → int + │ │ GVM_COMPUTE_FREEZE │ → 0/1 + │ └─────┬──────────────┘ + │ │ + │ ┌─────▼──────────────┐ + │ │ Find GPU processes │ + │ │ Match container PID │ + │ │ tree vs sysfs PIDs │ + │ └─────┬──────────────┘ + │ │ + │ ┌─────▼──────────────┐ + │ │ Write sysfs files │ + │ │ memory.limit │ + │ │ compute.priority │ + │ │ compute.freeze │ + │ └────────────────────┘ + │ + └─── (next container) +``` + +**Conversion logic (env var → sysfs):** + +``` +GVM_MEMORY_LIMIT=6g + → parse "6g" → 6000000000 (bytes) + → write "6000000000" to .../memory.limit + +GVM_MEMORY_PERCENTAGE=50 + → query GPU total memory (e.g., 24GB) + → compute 0.50 × 24000000000 = 12000000000 + → write "12000000000" to .../memory.limit + +GVM_COMPUTE_PRIORITY=2 + → write "2" to .../compute.priority + +GVM_COMPUTE_FREEZE=true + → write "1" to .../compute.freeze +``` + +**Priority between `GVM_MEMORY_LIMIT` and `GVM_MEMORY_PERCENTAGE`:** +- If both are set, `GVM_MEMORY_LIMIT` takes precedence (explicit byte value wins). +- `GVM_MEMORY_LIMIT_` (per-device) overrides `GVM_MEMORY_LIMIT` (global) for that device. + +### 3.5 Integration: Docker → cgroup API + +``` +Docker Layer cgroup API Layer +───────────── ──────────────── +Container starts with env vars + │ + ▼ +GVM Daemon reads env vars + │ + ▼ +Daemon finds container PID + │ + ▼ +Daemon walks /proc to find ───► /sys/kernel/debug/nvidia-uvm/ +child GPU processes processes// exists? + │ + ▼ +Daemon writes sysfs files ───► echo > .../memory.limit + echo > .../compute.priority + echo > .../compute.freeze +``` + +--- + +## 4. Layer 2: Docker Compose + +Docker Compose provides a **declarative YAML** interface for multi-container GPU workloads. It maps directly to `docker run` commands with GVM environment variables. + +### 4.1 Compose File Schema + +```yaml +services: + : + image: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: # Number of GPUs + capabilities: [gpu] + environment: + GVM_MEMORY_LIMIT: "" + GVM_COMPUTE_PRIORITY: "" + GVM_COMPUTE_FREEZE: "" + GVM_ENABLED: "" + GVM_DEBUG: "" +``` + +### 4.2 Example: Single Service + +```yaml +# docker-compose.yml +version: "3.8" + +services: + diffusion: + image: gvm-diffusion:latest + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + GVM_MEMORY_LIMIT: "6g" + GVM_COMPUTE_PRIORITY: "8" + volumes: + - ~/.cache/huggingface:/root/.cache/huggingface:ro +``` + +```bash +docker compose up -d +``` + +### 4.3 Example: Colocation (Inference + Training) + +```yaml +# colocation-compose.yml +version: "3.8" + +services: + # High-priority inference server + inference: + image: vllm-server:latest + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + GVM_MEMORY_LIMIT: "16g" + GVM_COMPUTE_PRIORITY: "2" + ports: + - "8000:8000" + + # Low-priority batch training + training: + image: training-image:latest + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + GVM_MEMORY_LIMIT: "8g" + GVM_COMPUTE_PRIORITY: "12" + volumes: + - ./data:/data + - ./checkpoints:/checkpoints +``` + +```bash +docker compose -f colocation-compose.yml up -d +``` + +### 4.4 Example: Multi-GPU Service + +```yaml +# multi-gpu-compose.yml +version: "3.8" + +services: + multi-gpu-training: + image: distributed-training:latest + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 2 + capabilities: [gpu] + environment: + GVM_MEMORY_LIMIT_0: "20g" + GVM_MEMORY_LIMIT_1: "20g" + GVM_COMPUTE_PRIORITY: "5" + volumes: + - ./data:/data +``` + +### 4.5 Example: Development vs Production Profiles + +```yaml +# docker-compose.yml +version: "3.8" + +services: + gpu-app: + image: my-gpu-app:latest + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + GVM_COMPUTE_PRIORITY: "8" + profiles: ["dev", "prod"] + + gpu-app-dev: + extends: + service: gpu-app + environment: + GVM_MEMORY_LIMIT: "4g" + GVM_DEBUG: "true" + profiles: ["dev"] + + gpu-app-prod: + extends: + service: gpu-app + environment: + GVM_MEMORY_LIMIT: "16g" + GVM_DEBUG: "false" + profiles: ["prod"] +``` + +```bash +# Development +docker compose --profile dev up -d + +# Production +docker compose --profile prod up -d +``` + +### 4.6 Integration: Docker Compose → Docker + +Docker Compose is a thin declarative layer. The mapping is direct: + +``` +Compose YAML Docker +──────────── ────── +services: + inference: + environment: docker run -d \ + GVM_MEMORY_LIMIT: "16g" → -e GVM_MEMORY_LIMIT=16g \ + GVM_COMPUTE_PRIORITY: "2" → -e GVM_COMPUTE_PRIORITY=2 \ + deploy: --gpus '"device=0"' \ + resources: vllm-server:latest + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + image: vllm-server:latest +``` + +No additional translation or daemon is required at this layer. Docker Compose calls Docker, which starts the container with environment variables. The GVM Docker Daemon (Layer 1) picks up from there. + +--- + +## 5. Layer 3: Kubernetes + +Kubernetes provides the highest-level abstraction: users declare GPU resource requirements in pod specs, and the system automatically schedules, allocates, and enforces them. + +### 5.1 Custom Resource Definitions + +GVM introduces custom resource names for Kubernetes: + +| Resource | Description | Unit | +|:---------|:------------|:-----| +| `gvm.io/gpu` | Number of GPUs requested | integer | +| `gvm.io/gpumem` | GPU memory limit | megabytes (MB) | +| `gvm.io/gpumem-percentage` | GPU memory as percentage | integer (1–100) | +| `gvm.io/priority` | Compute priority | integer (0–15, 0=highest) | + +### 5.2 Pod Spec API + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: inference-server +spec: + containers: + - name: vllm + image: vllm-server:latest + resources: + limits: + gvm.io/gpu: 1 + gvm.io/gpumem: 16000 # 16 GB in MB + gvm.io/priority: 2 # High priority + ports: + - containerPort: 8000 +``` + +### 5.3 Pod Annotations (Scheduling Hints) + +```yaml +metadata: + annotations: + # GPU selection + gvm.io/use-gputype: "NVIDIA A100" + gvm.io/nouse-gputype: "NVIDIA T4" + gvm.io/use-gpuuuid: "GPU-aaaa-bbbb" + + # Scheduling policy + gvm.io/gpu-scheduler-policy: "binpack" # or "spread" + gvm.io/node-scheduler-policy: "spread" # or "binpack" +``` + +### 5.4 Deployment Example: Colocation + +```yaml +# inference-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inference-server + namespace: gpu-workloads +spec: + replicas: 1 + selector: + matchLabels: + app: inference + template: + metadata: + labels: + app: inference + spec: + containers: + - name: vllm + image: vllm-server:latest + resources: + limits: + gvm.io/gpu: 1 + gvm.io/gpumem: 16000 + gvm.io/priority: 2 + ports: + - containerPort: 8000 +--- +# training-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: training-job + namespace: gpu-workloads +spec: + replicas: 1 + selector: + matchLabels: + app: training + template: + metadata: + labels: + app: training + spec: + containers: + - name: trainer + image: training-image:latest + resources: + limits: + gvm.io/gpu: 1 + gvm.io/gpumem: 8000 + gvm.io/priority: 12 + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: training-data +``` + +### 5.5 Kubernetes Components + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ Kubernetes Control Plane │ +│ │ +│ ┌──────────────┐ ┌──────────────────────┐ │ +│ │ API Server │────▶│ GVM Admission Webhook │ │ +│ │ │ │ (MutatingWebhook) │ │ +│ │ Pod spec: │ │ │ │ +│ │ gvm.io/gpu │ │ Validates resources │ │ +│ │ gvm.io/gpumem│ │ Injects defaults │ │ +│ │ gvm.io/... │ │ Adds env vars to pod │ │ +│ └──────┬───────┘ └──────────────────────┘ │ +│ │ │ +│ ┌──────▼───────┐ ┌──────────────────────┐ │ +│ │ Scheduler │────▶│ GVM Scheduler Extender│ │ +│ │ │ │ │ │ +│ │ Which node? │ │ Filters nodes by GPU │ │ +│ │ Which GPU? │ │ memory availability │ │ +│ │ │ │ Applies bin/spread │ │ +│ └──────┬───────┘ └──────────────────────┘ │ +│ │ │ +└─────────┼──────────────────────────────────────────────────────────┘ + │ + ▼ +┌────────────────────────────────────────────────────────────────────┐ +│ Kubernetes Node │ +│ │ +│ ┌──────────────┐ ┌──────────────────────┐ │ +│ │ Kubelet │────▶│ GVM Device Plugin │ │ +│ │ │ │ │ │ +│ │ Allocate() │ │ Maps gvm.io/gpu to │ │ +│ │ request │ │ physical GPU │ │ +│ │ │ │ │ │ +│ │ │ │ Injects into container│ │ +│ │ │ │ env vars: │ │ +│ │ │ │ GVM_MEMORY_LIMIT │ │ +│ │ │ │ GVM_COMPUTE_PRIORITY │ │ +│ │ │ │ NVIDIA_VISIBLE_DEVICES│ │ +│ └──────────────┘ └──────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────┐ │ +│ │ GVM Docker Daemon │ │ +│ │ (same as Layer 1) │ │ +│ │ │ │ +│ │ Reads container env vars │ │ +│ │ Writes sysfs controls │ │ +│ └──────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────┐ │ +│ │ GVM Kernel Module │ │ +│ │ (same as Layer 0) │ │ +│ └──────────────────────────────────────────┘ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +### 5.6 GVM Device Plugin (Allocate Behavior) + +When kubelet calls `Allocate()`, the GVM device plugin: + +1. Reads the pod's `gvm.io/*` resource requests +2. Selects a physical GPU based on scheduler decision +3. Returns a `ContainerAllocateResponse` with: + +```go +response.Envs = map[string]string{ + "GVM_MEMORY_LIMIT": fmt.Sprintf("%dm", req.Gpumem), // e.g. "16000m" + "GVM_COMPUTE_PRIORITY": fmt.Sprintf("%d", req.Priority), // e.g. "2" + "NVIDIA_VISIBLE_DEVICES": selectedGPU.UUID, +} +``` + +The device plugin does **not** write sysfs directly. It injects environment variables into the container, and the GVM Docker Daemon (running on the same node) handles the rest. This maintains the layered architecture. + +### 5.7 Integration: Kubernetes → Docker + +``` +Kubernetes Layer Docker Layer +──────────────── ──────────── +Pod spec: + gvm.io/gpumem: 16000 + gvm.io/priority: 2 + │ + ▼ +GVM Admission Webhook + (validates, injects defaults) + │ + ▼ +GVM Scheduler Extender + (selects node + GPU) + │ + ▼ +Kubelet → GVM Device Plugin + Allocate() returns: Container starts with: + GVM_MEMORY_LIMIT=16000m → -e GVM_MEMORY_LIMIT=16000m + GVM_COMPUTE_PRIORITY=2 → -e GVM_COMPUTE_PRIORITY=2 + NVIDIA_VISIBLE_DEVICES=GPU-x → --gpus device=GPU-x + │ + ▼ ▼ + GVM Docker Daemon (Layer 1) + reads env vars, writes sysfs + │ + ▼ + cgroup API (Layer 0) +``` + +--- + +## 6. Layer Integration: Top-Down Request Flow + +This section traces a complete request from the highest abstraction (Kubernetes) down to the kernel. + +### 6.1 Full Path: Kubernetes Pod → GPU Kernel Enforcement + +``` +USER creates Pod YAML: + resources.limits: + gvm.io/gpu: 1 + gvm.io/gpumem: 16000 + gvm.io/priority: 2 + │ + ▼ +[K8s API Server] ──► [GVM Webhook] + │ │ + │ Validates: gpumem ≤ total GPU memory + │ Injects defaults for missing fields + │ (e.g., default priority = 8 if not set) + │ + ▼ +[K8s Scheduler] ──► [GVM Scheduler Extender] + │ │ + │ Filters nodes: which nodes have GPUs with ≥16GB free? + │ Scores nodes: binpack or spread policy + │ Selects: node-3, GPU-0 + │ + ▼ +[Kubelet on node-3] ──► [GVM Device Plugin] + │ │ + │ Allocate() called with device IDs + │ Returns ContainerAllocateResponse: + │ Envs: { + │ "GVM_MEMORY_LIMIT": "16000m", + │ "GVM_COMPUTE_PRIORITY": "2", + │ "NVIDIA_VISIBLE_DEVICES": "GPU-aaaa-bbbb" + │ } + │ + ▼ +[Container Runtime (containerd/docker)] + │ Starts container with env vars + │ GPU process eventually starts inside container + │ + ▼ +[GVM Docker Daemon] (running on node-3) + │ Polls containers, finds GVM_* env vars + │ Discovers GPU PID via /proc + nvidia-uvm + │ Parses: "16000m" → 16000000000 bytes + │ Parses: "2" → integer 2 + │ + ▼ +[GVM Kernel Module] (sysfs writes) + │ echo 16000000000 > .../PID/0/memory.limit + │ echo 2 > .../PID/0/compute.priority + │ + ▼ +[GPU Hardware] + Memory allocation enforced at page-fault level + Compute scheduling uses priority 2 timeslice +``` + +### 6.2 Full Path: Docker Compose → GPU Kernel Enforcement + +``` +USER writes docker-compose.yml: + services: + inference: + environment: + GVM_MEMORY_LIMIT: "16g" + GVM_COMPUTE_PRIORITY: "2" + │ + ▼ +[docker compose up -d] + │ Translates YAML to docker run command + │ docker run -d -e GVM_MEMORY_LIMIT=16g \ + │ -e GVM_COMPUTE_PRIORITY=2 --gpus all ... + │ + ▼ +[Docker Engine] + │ Starts container with env vars + │ + ▼ +[GVM Docker Daemon] + │ Reads env vars from container + │ Parses: "16g" → 16000000000 bytes + │ Finds GPU PIDs + │ + ▼ +[GVM Kernel Module] + │ echo 16000000000 > .../PID/0/memory.limit + │ echo 2 > .../PID/0/compute.priority + │ + ▼ +[GPU Hardware] +``` + +### 6.3 Full Path: Docker Run → GPU Kernel Enforcement + +``` +USER runs: + docker run -d --gpus all \ + -e GVM_MEMORY_LIMIT=6g \ + -e GVM_COMPUTE_PRIORITY=5 \ + my-app + │ + ▼ +[Docker Engine] + │ Container starts, GPU process initializes + │ + ▼ +[GVM Docker Daemon] + │ Detects container via docker inspect + │ Reads: GVM_MEMORY_LIMIT=6g, GVM_COMPUTE_PRIORITY=5 + │ Parses: "6g" → 6000000000 bytes + │ Matches container PID tree to nvidia-uvm PIDs + │ + ▼ +[GVM Kernel Module] + │ write(/.../PID/0/memory.limit, "6000000000") + │ write(/.../PID/0/compute.priority, "5") + │ + ▼ +[GPU Hardware] +``` + +--- + +## 7. Bottom-Up Visibility + +Each layer can also **read** GPU state from below. This enables monitoring and observability. + +### 7.1 cgroup API (Layer 0) → Monitoring + +```bash +# Direct sysfs reads +cat /sys/kernel/debug/nvidia-uvm/processes//0/memory.current +cat /sys/kernel/debug/nvidia-uvm/processes//0/memory.swap.current +cat /sys/kernel/debug/nvidia-uvm/processes//0/gcgroup.stat +``` + +### 7.2 Docker (Layer 1) → Container GPU Stats + +The GVM Daemon can expose per-container GPU stats: + +```bash +# Future: gvm-docker-daemon exposes stats endpoint +curl http://localhost:9090/api/v1/containers +``` + +```json +[ + { + "container_id": "abc123", + "container_name": "inference", + "gpu_processes": [ + { + "pid": 12345, + "gpu_index": 0, + "memory_limit": 16000000000, + "memory_current": 12500000000, + "memory_swap_current": 0, + "compute_priority": 2, + "compute_frozen": false + } + ] + } +] +``` + +### 7.3 Docker Compose (Layer 2) → Service GPU Stats + +```bash +# Future: per-service stats +docker compose exec inference gvm-stats +# Shows: memory 12.5G / 16G (78%), priority 2, swap 0 +``` + +### 7.4 Kubernetes (Layer 3) → Pod GPU Metrics + +The GVM monitoring component exports Prometheus metrics: + +``` +# HELP gvm_gpu_memory_usage_bytes Current GPU memory usage per container +# TYPE gvm_gpu_memory_usage_bytes gauge +gvm_gpu_memory_usage_bytes{pod="inference-server-abc",namespace="gpu-workloads",gpu="0"} 12500000000 + +# HELP gvm_gpu_memory_limit_bytes GPU memory limit per container +# TYPE gvm_gpu_memory_limit_bytes gauge +gvm_gpu_memory_limit_bytes{pod="inference-server-abc",namespace="gpu-workloads",gpu="0"} 16000000000 + +# HELP gvm_gpu_compute_priority Compute priority per container +# TYPE gvm_gpu_compute_priority gauge +gvm_gpu_compute_priority{pod="inference-server-abc",namespace="gpu-workloads",gpu="0"} 2 + +# HELP gvm_gpu_memory_swap_bytes GPU memory swapped to host per container +# TYPE gvm_gpu_memory_swap_bytes gauge +gvm_gpu_memory_swap_bytes{pod="inference-server-abc",namespace="gpu-workloads",gpu="0"} 0 +``` + +--- + +## Appendix A: Complete Environment Variable Reference + +| Variable | Layer | Required | Default | Description | +|:---------|:------|:---------|:--------|:------------| +| `GVM_MEMORY_LIMIT` | Docker+ | No | unlimited | GPU memory limit. Accepts `b/k/m/g/t` or raw bytes. | +| `GVM_MEMORY_LIMIT_` | Docker+ | No | `GVM_MEMORY_LIMIT` | Per-device override for GPU index N. | +| `GVM_MEMORY_PERCENTAGE` | Docker+ | No | — | GPU memory as % of total. Ignored if `GVM_MEMORY_LIMIT` is set. | +| `GVM_COMPUTE_PRIORITY` | Docker+ | No | `8` | Priority 0–15. 0=highest, 15=lowest. | +| `GVM_COMPUTE_FREEZE` | Docker+ | No | `false` | Freeze GPU compute. `true` or `false`. | +| `GVM_ENABLED` | Docker+ | No | `true` | Enable/disable GVM controls for this container. | +| `GVM_DEBUG` | Docker+ | No | `false` | Enable verbose logging for this container. | + +## Appendix B: Kubernetes Resource Reference + +| Resource | Type | Description | +|:---------|:-----|:------------| +| `gvm.io/gpu` | integer | Number of GPUs requested. | +| `gvm.io/gpumem` | integer (MB) | GPU memory limit in megabytes. | +| `gvm.io/gpumem-percentage` | integer (1–100) | GPU memory as percentage. | +| `gvm.io/priority` | integer (0–15) | Compute priority. 0=highest. | + +## Appendix C: Layer Dependency Summary + +``` +Layer 3 (Kubernetes) + │ + │ Requires: GVM Device Plugin, GVM Scheduler Extender, + │ GVM Admission Webhook + │ Produces: Container env vars (GVM_MEMORY_LIMIT, GVM_COMPUTE_PRIORITY) + │ Consumes from Layer 2/1: GVM Docker Daemon on each node + │ + ▼ +Layer 2 (Docker Compose) + │ + │ Requires: Docker Engine, docker-compose CLI + │ Produces: docker run commands with GVM_* env vars + │ Consumes from Layer 1: Docker Engine + GVM Docker Daemon + │ + ▼ +Layer 1 (Docker) + │ + │ Requires: Docker Engine, GVM Docker Daemon, NVIDIA Container Toolkit + │ Produces: Container processes with env vars + │ Consumes from Layer 0: Reads/writes GVM sysfs files + │ + ▼ +Layer 0 (cgroup API) + │ + │ Requires: GVM kernel modules loaded + │ Produces: sysfs files under /sys/kernel/debug/nvidia-uvm/processes/ + │ Consumes: GPU hardware via nvidia-uvm driver + │ + ▼ +GPU Hardware +``` + +## Appendix D: Implementation Priority + +| Phase | Scope | Status | +|:------|:------|:-------| +| Phase 0 | cgroup API (kernel module) | ✅ Done (GVM kernel modules) | +| Phase 1 | Docker + GVM Daemon (basic) | ✅ Done (basic `GVM_MEMORY_LIMIT`, `GVM_COMPUTE_PRIORITY`) | +| Phase 2 | Docker enhanced (unit parsing, per-device, percentage, freeze) | 🔲 Next | +| Phase 3 | Docker Compose examples and testing | 🔲 Planned | +| Phase 4 | Kubernetes Device Plugin + Webhook | 🔲 Planned | +| Phase 5 | Kubernetes Scheduler Extender | 🔲 Planned | +| Phase 6 | Monitoring + Prometheus metrics | 🔲 Planned | diff --git a/gvm-docker/HAMI_ANALYSIS.md b/gvm-docker/HAMI_ANALYSIS.md new file mode 100644 index 00000000..8830ee58 --- /dev/null +++ b/gvm-docker/HAMI_ANALYSIS.md @@ -0,0 +1,302 @@ +# HAMi Analysis: Docker APIs for GPU & Implications for GVM + +## 1. HAMi Overview + +HAMi (Heterogeneous AI Computing Virtualization Middleware) is a CNCF sandbox project for GPU sharing on Kubernetes. It consists of three main components: + +1. **HAMi Scheduler** — K8s scheduler extender for GPU-aware pod placement +2. **HAMi Device Plugin** — K8s device plugin that intercepts container creation and injects GPU controls +3. **HAMi-core** (`libvgpu.so`) — C shared library that hooks CUDA API calls inside the container to enforce limits + +## 2. HAMi's Docker/Container APIs for GPU + +### 2.1 Kubernetes Resource API (Primary Interface) + +HAMi's main API is through Kubernetes pod resource requests: + +```yaml +resources: + limits: + nvidia.com/gpu: 1 # Number of physical GPUs needed + nvidia.com/gpumem: 3000 # GPU memory limit in MB + nvidia.com/gpumem-percentage: 50 # GPU memory as percentage of total + nvidia.com/gpucores: 50 # GPU SM core utilization cap (0-100%) + nvidia.com/priority: 5 # Task scheduling priority +``` + +### 2.2 Environment Variables (Container-Level API) + +These are injected into containers by the device plugin and consumed by HAMi-core (`libvgpu.so`): + +| Environment Variable | Description | Example | +|---|---|---| +| `CUDA_DEVICE_MEMORY_LIMIT_` | Memory limit per device N (in MB suffix `m`) | `3000m` | +| `CUDA_DEVICE_SM_LIMIT` | SM utilization percentage cap | `50` | +| `CUDA_DEVICE_MEMORY_SHARED_CACHE` | Path for shared memory cache file | `/path/to/cache` | +| `CUDA_OVERSUBSCRIBE` | Allow virtual memory beyond physical | `true` | +| `GPU_CORE_UTILIZATION_POLICY` | Core limit enforcement policy | `default`, `force`, `disable` | +| `CUDA_DISABLE_CONTROL` | Disable all HAMi-core enforcement | `true`, `false` | +| `LIBCUDA_LOG_LEVEL` | Logging verbosity | `0`=error, `1`=warn, `3`=info, `4`=debug | + +### 2.3 Pod Annotations (Scheduling Hints) + +```yaml +annotations: + nvidia.com/use-gputype: "Tesla V100-PCIE-32GB" # Require specific GPU type + nvidia.com/nouse-gputype: "NVIDIA A10" # Exclude GPU type + nvidia.com/use-gpuuuid: "GPU-AAA,GPU-BBB" # Pin to specific GPU UUIDs + nvidia.com/nouse-gpuuuid: "GPU-CCC" # Exclude specific GPUs + nvidia.com/vgpu-mode: "hami-core" # Virtualization mode (hami-core | mig | mps) + hami.io/gpu-scheduler-policy: "binpack" # GPU-level scheduling (binpack | spread) + hami.io/node-scheduler-policy: "spread" # Node-level scheduling (binpack | spread) +``` + +### 2.4 Standalone Docker Usage (Without Kubernetes) + +HAMi-core can be used directly with Docker by manually setting environment variables: + +```bash +docker run \ + --device /dev/nvidia0:/dev/nvidia0 \ + --device /dev/nvidia-uvm:/dev/nvidia-uvm \ + --device /dev/nvidiactl:/dev/nvidiactl \ + -e CUDA_DEVICE_MEMORY_LIMIT=2g \ + -e CUDA_DEVICE_SM_LIMIT=50 \ + -e LD_PRELOAD=/libvgpu/build/libvgpu.so \ + my-gpu-image +``` + +### 2.5 Global Configuration (Cluster-Level) + +Via ConfigMap `hami-scheduler-device`: + +| Config Key | Description | Default | +|---|---|---| +| `nvidia.deviceMemoryScaling` | Memory overcommit ratio (>1 = virtual memory) | `1` | +| `nvidia.deviceSplitCount` | Max tasks per GPU | `10` | +| `nvidia.defaultMem` | Default memory per task (MB, 0=100%) | `0` | +| `nvidia.defaultCores` | Default core % per task (0=any GPU with enough mem) | `0` | +| `nvidia.disablecorelimit` | Disable core limitation | `false` | +| `nvidia.resourcePriorityName` | Custom resource name for priority | `nvidia.com/priority` | + +## 3. How HAMi Implements GPU Controls + +### 3.1 Architecture Flow + +``` +Pod Spec (resource requests) + │ + ▼ +HAMi Webhook (mutating admission) + │ Validates and enriches pod spec + ▼ +HAMi Scheduler Extender + │ Selects optimal node/GPU based on policy + ▼ +HAMi Device Plugin (on node) + │ Intercepts kubelet Allocate() call + │ Injects into container: + │ - Environment vars (CUDA_DEVICE_MEMORY_LIMIT_0, etc.) + │ - Volume mount: libvgpu.so + │ - Volume mount: /etc/ld.so.preload (forces LD_PRELOAD) + │ - Volume mount: shared cache directory + ▼ +Container starts with LD_PRELOAD=libvgpu.so + │ + ▼ +HAMi-core (libvgpu.so) inside container + │ Hooks dlsym() to intercept all CUDA driver calls + │ Intercepts: cuMemAlloc, cuMemAllocManaged, cuLaunchKernel, etc. + │ Enforces memory limit by tracking allocations + │ Enforces SM limit via time-slicing + │ Hooks NVML to report virtual limits (nvidia-smi shows virtual mem) + ▼ +CUDA Driver (libcuda.so) → GPU Hardware +``` + +### 3.2 Key Implementation Details + +**Device Plugin (`server.go` Allocate method):** +```go +// Injects per-device memory limits +for i, dev := range devreq { + limitKey := fmt.Sprintf("CUDA_DEVICE_MEMORY_LIMIT_%v", i) + response.Envs[limitKey] = fmt.Sprintf("%vm", dev.Usedmem) +} +response.Envs["CUDA_DEVICE_SM_LIMIT"] = fmt.Sprint(devreq[0].Usedcores) + +// Mounts libvgpu.so into container +response.Mounts = append(response.Mounts, + &Mount{ContainerPath: "/path/libvgpu.so", HostPath: GetLibPath(), ReadOnly: true}, +) + +// Mounts ld.so.preload to force library loading +response.Mounts = append(response.Mounts, + &Mount{ContainerPath: "/etc/ld.so.preload", HostPath: "ld.so.preload", ReadOnly: true}, +) +``` + +**HAMi-core (`libvgpu.c`):** +- Hooks `dlsym()` itself to intercept all CUDA symbol lookups +- When a CUDA function like `cuMemAlloc` is called, it goes through HAMi-core first +- HAMi-core tracks total allocations and rejects if over limit +- For SM limits, implements userspace time-slicing to cap utilization + +## 4. GVM vs HAMi: Comparison + +| Aspect | HAMi | GVM | +|--------|------|-----| +| **Enforcement Layer** | Userspace (LD_PRELOAD CUDA hook) | Kernel (nvidia-uvm module, sysfs) | +| **Memory Control** | CUDA API interception, tracks `cuMemAlloc` | Kernel page-level control via `memory.limit` | +| **Compute Control** | Userspace time-slicing of SM | Kernel scheduler `compute.priority` (0-15) | +| **Bypass Resistance** | Can be bypassed by direct driver calls or removing LD_PRELOAD | Kernel-level, **cannot be bypassed** | +| **Overhead** | Adds latency to every CUDA API call | Minimal kernel-level overhead | +| **Memory Overcommit** | Yes (`CUDA_OVERSUBSCRIBE`) | Yes (kernel-level swap) | +| **Visibility** | Hooks nvidia-smi to show virtual limits | Transparent (real kernel stats) | +| **Platform** | Primarily Kubernetes, standalone Docker possible | Standalone, Docker daemon | +| **Multi-device** | Per-device limits (`CUDA_DEVICE_MEMORY_LIMIT_0`, `_1`, etc.) | Per-process per-GPU via sysfs | +| **Priority** | `nvidia.com/priority` (scheduling only) | `compute.priority` (kernel scheduler, real-time) | +| **Installation** | Helm chart (K8s), or manual LD_PRELOAD | Kernel module + daemon | + +### Key Advantages of GVM over HAMi: +1. **Kernel-level enforcement** — Cannot be bypassed by applications +2. **Lower overhead** — No per-API-call interception +3. **Real compute priority** — Kernel scheduler vs userspace time-slicing +4. **Simpler in-container setup** — No LD_PRELOAD or library injection needed +5. **Memory swap support** — Kernel-level page migration + +### What HAMi Has That GVM Could Adopt: +1. **Clean Docker environment variable API** — Standard naming convention +2. **Per-device granularity** — `CUDA_DEVICE_MEMORY_LIMIT_0`, `_1`, etc. +3. **Percentage-based limits** — `gpumem-percentage` alongside absolute values +4. **Scheduling policies** — binpack/spread at node and GPU level +5. **Disable/debug controls** — Easy toggle to disable enforcement +6. **Memory overcommit ratio** — Configurable oversubscription + +## 5. Proposed GVM Docker API Design + +Based on HAMi's patterns but leveraging GVM's kernel-level capabilities: + +### 5.1 Container Environment Variables + +```bash +# Memory control +GVM_MEMORY_LIMIT=6g # Memory limit (supports g/m/k/bytes) +GVM_MEMORY_LIMIT_0=6g # Per-device memory limit (device 0) +GVM_MEMORY_LIMIT_1=4g # Per-device memory limit (device 1) +GVM_MEMORY_PERCENTAGE=50 # Memory as percentage of total + +# Compute control +GVM_COMPUTE_PRIORITY=8 # Compute priority (0-15, higher = more priority) + +# Memory management +GVM_MEMORY_SWAP=true # Enable kernel-level memory swap +GVM_MEMORY_OVERCOMMIT=1.5 # Overcommit ratio + +# Control +GVM_ENABLED=true # Enable/disable GVM controls (default: true) +GVM_DEBUG=true # Enable debug logging + +# Freeze/throttle (advanced) +GVM_COMPUTE_FREEZE=false # Freeze compute for this container +``` + +### 5.2 Docker Run Examples + +```bash +# Simple: 6GB memory, medium priority +docker run -d --gpus all \ + --env GVM_MEMORY_LIMIT=6g \ + --env GVM_COMPUTE_PRIORITY=8 \ + my-gpu-app + +# Colocation: high-priority inference + low-priority training +docker run -d --gpus all \ + --env GVM_MEMORY_LIMIT=12g \ + --env GVM_COMPUTE_PRIORITY=15 \ + --name inference-server \ + vllm-server + +docker run -d --gpus all \ + --env GVM_MEMORY_LIMIT=8g \ + --env GVM_COMPUTE_PRIORITY=3 \ + --name training-job \ + training-image + +# Percentage-based limit +docker run -d --gpus all \ + --env GVM_MEMORY_PERCENTAGE=50 \ + --env GVM_COMPUTE_PRIORITY=10 \ + my-gpu-app + +# Multi-GPU with per-device limits +docker run -d --gpus all \ + --env GVM_MEMORY_LIMIT_0=8g \ + --env GVM_MEMORY_LIMIT_1=4g \ + --env GVM_COMPUTE_PRIORITY=7 \ + multi-gpu-app +``` + +### 5.3 GVM Daemon API + +The `gvm-docker-daemon` monitors containers and applies controls: + +``` +Daemon reads container env vars → Maps to GVM sysfs writes: + +GVM_MEMORY_LIMIT=6g → write "6000000000" to .../memory.limit +GVM_COMPUTE_PRIORITY=8 → write "8" to .../compute.priority +GVM_COMPUTE_FREEZE=true → write "1" to .../compute.freeze +GVM_MEMORY_SWAP=true → write "1" to .../memory.swap_enabled +``` + +### 5.4 Comparison: HAMi API vs Proposed GVM API + +| HAMi API | Proposed GVM API | Notes | +|----------|-----------------|-------| +| `nvidia.com/gpumem: 3000` | `GVM_MEMORY_LIMIT=3g` | GVM supports human-readable units | +| `nvidia.com/gpumem-percentage: 50` | `GVM_MEMORY_PERCENTAGE=50` | Same concept | +| `nvidia.com/gpucores: 50` | `GVM_COMPUTE_PRIORITY=8` | GVM uses priority (0-15) not % | +| `nvidia.com/priority: 5` | `GVM_COMPUTE_PRIORITY=5` | Direct mapping | +| `CUDA_DEVICE_MEMORY_LIMIT_0=3000m` | `GVM_MEMORY_LIMIT_0=3g` | Same pattern, cleaner naming | +| `CUDA_DEVICE_SM_LIMIT=50` | *(no equivalent)* | GVM uses priority, not SM capping | +| `CUDA_DISABLE_CONTROL=true` | `GVM_ENABLED=false` | Same concept | +| `CUDA_OVERSUBSCRIBE=true` | `GVM_MEMORY_SWAP=true` | GVM does kernel-level swap | + +## 6. Implementation Plan + +### Phase 1: Enhanced Docker Daemon (Current) +- [x] Basic daemon with `GVM_MEMORY_LIMIT` and `GVM_COMPUTE_PRIORITY` +- [ ] Add human-readable unit parsing (g/m/k) +- [ ] Add per-device limits (`GVM_MEMORY_LIMIT_0`, `GVM_MEMORY_LIMIT_1`) +- [ ] Add percentage-based limits (`GVM_MEMORY_PERCENTAGE`) +- [ ] Add `GVM_ENABLED` toggle +- [ ] Add `GVM_DEBUG` logging level + +### Phase 2: Advanced Controls +- [ ] Memory swap support (`GVM_MEMORY_SWAP`) +- [ ] Compute freeze (`GVM_COMPUTE_FREEZE`) +- [ ] Dynamic control updates (change limits on running containers) +- [ ] Container lifecycle tracking (cleanup on exit) + +### Phase 3: Kubernetes Integration +- [ ] K8s device plugin for GVM +- [ ] Custom resource definitions (`gvm.io/gpumem`, `gvm.io/priority`) +- [ ] Scheduler extender for GVM-aware placement +- [ ] Webhook for automatic env var injection + +### Phase 4: Monitoring & Observability +- [ ] Prometheus metrics exporter +- [ ] Per-container GPU memory usage tracking +- [ ] Dashboard (Grafana) +- [ ] Alerting on memory pressure + +## 7. Recommendation + +**We should adopt a similar environment variable API pattern to HAMi's**, but with GVM-specific naming (`GVM_` prefix) and leveraging GVM's unique kernel-level capabilities: + +1. **Yes, adopt**: Environment variable convention, per-device limits, percentage-based limits, enable/disable toggle +2. **No, skip**: LD_PRELOAD mechanism (GVM doesn't need it — kernel-level), CUDA API hooking, userspace time-slicing +3. **GVM-unique**: Compute priority (real kernel scheduler), memory swap, compute freeze — these are GVM advantages over HAMi + +The key insight is that **HAMi does everything in userspace via CUDA hooking**, while **GVM does it at kernel level**. This means GVM's Docker integration is simpler (no library injection needed) and stronger (cannot be bypassed), but we should match HAMi's clean API ergonomics. diff --git a/gvm-docker/Makefile b/gvm-docker/Makefile new file mode 100644 index 00000000..0c4631e7 --- /dev/null +++ b/gvm-docker/Makefile @@ -0,0 +1,49 @@ +.PHONY: build build-daemon build-runc install test clean uninstall + +INSTALL_PATH=/usr/local/bin + +build: build-daemon build-runc + +build-daemon: + @echo "Building gvm-daemon..." + go build -o gvm-daemon ./cmd/gvm-daemon/ + +build-runc: + @echo "Building gvm-runc..." + go build -o gvm-runc ./cmd/gvm-runc/ + +install: build + @echo "Installing binaries to $(INSTALL_PATH)..." + sudo cp gvm-daemon $(INSTALL_PATH)/gvm-daemon + sudo cp gvm-runc $(INSTALL_PATH)/gvm-runc + sudo chmod +x $(INSTALL_PATH)/gvm-daemon $(INSTALL_PATH)/gvm-runc + @echo "" + @echo "Installation complete!" + @echo "" + @echo "=== GVM Docker Daemon ===" + @echo "Start the daemon:" + @echo " sudo gvm-daemon" + @echo "" + @echo "=== GVM Runc Wrapper (alternative) ===" + @echo "Add this to /etc/docker/daemon.json:" + @echo '{' + @echo ' "runtimes": {' + @echo ' "gvm": {' + @echo ' "path": "$(INSTALL_PATH)/gvm-runc"' + @echo ' }' + @echo ' }' + @echo '}' + @echo "" + @echo "Then restart Docker: sudo systemctl restart docker" + +test: + @echo "Running tests..." + go test -v ./... + +clean: + @echo "Cleaning..." + rm -f gvm-daemon gvm-runc + +uninstall: + @echo "Uninstalling..." + sudo rm -f $(INSTALL_PATH)/gvm-daemon $(INSTALL_PATH)/gvm-runc diff --git a/gvm-docker/README.md b/gvm-docker/README.md new file mode 100644 index 00000000..2ce33191 --- /dev/null +++ b/gvm-docker/README.md @@ -0,0 +1,335 @@ +# GVM-Docker Integration + +Docker and Docker Compose integration for [GVM](https://github.com/ovg-project/GVM) (GPU Virtualization Manager). Provides kernel-level GPU memory limits, compute priority scheduling, and compute freeze via simple environment variables. + +## Features + +- **GPU Memory Limits** — Cap GPU memory per container (`GVM_MEMORY_LIMIT=6g`) +- **Per-Device Limits** — Set different limits per GPU (`GVM_MEMORY_LIMIT_0=8g`) +- **Percentage Limits** — Portable across GPU models (`GVM_MEMORY_PERCENTAGE=50`) +- **Compute Priority** — Kernel-level GPU scheduling (0=highest, 15=lowest) +- **Compute Freeze** — Pause/resume GPU execution for preemption +- **Human-Readable Units** — Supports `g`, `m`, `k`, `b` suffixes (SI units) +- **Docker Compose Ready** — Declarative YAML for multi-container GPU workloads +- **Kernel Enforcement** — Cannot be bypassed from userspace (unlike LD_PRELOAD approaches) + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Docker Container │ +│ ┌────────────────────────────────────────────────────┐ │ +│ │ GPU Application (e.g., vLLM, diffusion, training) │ │ +│ │ Environment: GVM_MEMORY_LIMIT=6g │ │ +│ │ Environment: GVM_COMPUTE_PRIORITY=2 │ │ +│ └────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + │ GPU Process (CUDA/UVM) + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ GVM Daemon or GVM Runc Wrapper (Host) │ +│ - Reads env vars from containers → parses units │ +│ - Discovers GPU PIDs via /sys/kernel/debug/nvidia-uvm/ │ +│ - Writes controls to sysfs per-process, per-GPU │ +└─────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ GVM Kernel Module (sysfs interface) │ +│ /sys/kernel/debug/nvidia-uvm/processes/$PID/$GPU/ │ +│ ├── memory.limit (read/write, bytes) │ +│ ├── memory.current (read-only, bytes) │ +│ ├── memory.swap.current (read-only, bytes) │ +│ ├── compute.priority (read/write, 0–15) │ +│ ├── compute.freeze (read/write, 0 or 1) │ +│ └── gcgroup.stat (read-only, stats) │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Project Structure + +``` +gvm-docker/ +├── cmd/ +│ ├── gvm-daemon/ # Host daemon — monitors Docker containers +│ └── gvm-runc/ # OCI runtime wrapper (alternative to daemon) +├── pkg/ +│ └── gvm/ # Shared library: config parsing, sysfs I/O, process discovery +│ ├── config.go # Config struct, env var parsing, memory unit parsing +│ ├── config_test.go # Unit tests (43 tests) +│ ├── sysfs.go # Sysfs read/write, GPU memory query +│ └── process.go # GPU process discovery via /proc +├── examples/ +│ ├── docker-compose/ # Compose file examples (4 scenarios) +│ └── diffusion/ # Example GPU workload Dockerfile +├── Makefile +└── go.mod +``` + +## Prerequisites + +1. **GVM kernel modules** installed and loaded: + + ```bash + sudo modprobe ecdh_generic + sudo insmod kernel-open/nvidia.ko + sudo insmod kernel-open/nvidia-uvm.ko + ``` + +2. **Docker** with **NVIDIA Container Toolkit**: + + ```bash + sudo apt-get install -y docker.io nvidia-container-toolkit + sudo nvidia-ctk runtime configure --runtime=docker + sudo systemctl restart docker + ``` + +3. **Go 1.21+** (for building from source) + +## Installation + +```bash +cd gvm-docker +make build +sudo make install # installs to /usr/local/bin +``` + +## Deployment Options + +### Option A: GVM Daemon (Recommended) + +The daemon polls Docker every 5 seconds, automatically applying GVM controls to any container with `GVM_*` environment variables. + +```bash +# Foreground with logging +sudo gvm-daemon 2>&1 | tee /tmp/gvm-daemon.log + +# Background +sudo gvm-daemon > /tmp/gvm-daemon.log 2>&1 & +``` + +At startup the daemon queries `nvidia-smi` for total GPU memory, enabling `GVM_MEMORY_PERCENTAGE` to work automatically. + +### Option B: GVM Runc Wrapper + +Register `gvm-runc` as a custom Docker OCI runtime. Controls are applied synchronously after each container starts. + +```bash +sudo tee /etc/docker/daemon.json <` | Per-device memory limit for GPU index N | `GVM_MEMORY_LIMIT` | `GVM_MEMORY_LIMIT_0=8g` | +| `GVM_MEMORY_PERCENTAGE` | Memory as percentage of total GPU memory (1–100) | — | `50` | +| `GVM_COMPUTE_PRIORITY` | Compute priority (0–15, 0=highest, 15=lowest) | `8` | `2` | +| `GVM_COMPUTE_FREEZE` | Freeze GPU compute (`true`/`false`) | `false` | `true` | +| `GVM_ENABLED` | Enable/disable GVM controls | `true` | `false` | +| `GVM_DEBUG` | Enable verbose debug logging | `false` | `true` | + +### Memory Limit Priority + +When multiple memory settings are specified, they resolve in this order: + +1. **Per-device** (`GVM_MEMORY_LIMIT_0`) — highest priority +2. **Global** (`GVM_MEMORY_LIMIT`) — fallback +3. **Percentage** (`GVM_MEMORY_PERCENTAGE`) — lowest priority, requires GPU memory query + +### Memory Unit Parsing + +Uses SI units (powers of 10). Decimal values supported (e.g., `1.5g`). + +| Suffix | Multiplier | Example | +| :------- | :--------- | :------------------- | +| `t` | × 10¹² | `1t` = 1 TB | +| `g` | × 10⁹ | `6g` = 6 GB | +| `m` | × 10⁶ | `6000m` = 6 GB | +| `k` | × 10³ | `1500k` = 1.5 MB | +| `b` | × 1 | `6000000000b` = 6 GB | +| _(none)_ | × 1 | `6000000000` = 6 GB | + +## Verifying Controls + +After the daemon applies controls, verify via sysfs: + +```bash +# Find GPU process PIDs +sudo ls /sys/kernel/debug/nvidia-uvm/processes/ + +# Check a specific PID's controls +export PID= +sudo cat /sys/kernel/debug/nvidia-uvm/processes/$PID/0/memory.limit +sudo cat /sys/kernel/debug/nvidia-uvm/processes/$PID/0/compute.priority +sudo cat /sys/kernel/debug/nvidia-uvm/processes/$PID/0/memory.current +``` + +## Systemd Service + +For production, run the daemon as a systemd service: + +```ini +# /etc/systemd/system/gvm-daemon.service +[Unit] +Description=GVM Docker Daemon +After=docker.service +Requires=docker.service + +[Service] +Type=simple +ExecStart=/usr/local/bin/gvm-daemon +Restart=always +RestartSec=5 +StandardOutput=append:/var/log/gvm-daemon.log +StandardError=append:/var/log/gvm-daemon.log + +[Install] +WantedBy=multi-user.target +``` + +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now gvm-daemon +sudo systemctl status gvm-daemon +``` + +## Troubleshooting + +### Container can't see GPU + +`torch.cuda.is_available()` returns `False`: + +```bash +sudo nvidia-ctk runtime configure --runtime=docker +sudo systemctl restart docker +``` + +### Memory limit not applied + +`memory.limit` shows `18446744073709551615` (unlimited): + +1. **Daemon not running** — `ps aux | grep gvm-daemon` +2. **No GPU process yet** — GVM controls apply only to CUDA/UVM processes, not `nvidia-smi`. Wait for the application to initialize CUDA. +3. **Missing env vars** — `docker inspect | grep GVM_` + +### Out of memory errors + +Multiple containers exceeding GPU memory: + +```bash +docker stop $(docker ps -q) # stop all +nvidia-smi # verify GPU free +# re-launch with adjusted limits +``` + +## Development + +```bash +make build # Build both binaries +make test # Run unit tests (43 tests) +make install # Install to /usr/local/bin +make clean # Remove build artifacts +``` + +## Further Reading + +- **[API_DESIGN.md](API_DESIGN.md)** — Layered API design (cgroup → Docker → Compose → Kubernetes) +- **[HAMI_ANALYSIS.md](HAMI_ANALYSIS.md)** — Comparison with HAMi's userspace approach + +## License + +Same as GVM project diff --git a/gvm-docker/cmd/gvm-daemon/main.go b/gvm-docker/cmd/gvm-daemon/main.go new file mode 100644 index 00000000..593cb042 --- /dev/null +++ b/gvm-docker/cmd/gvm-daemon/main.go @@ -0,0 +1,238 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "time" + + "github.com/ovg-project/gvm-docker/pkg/gvm" +) + +const ( + pollInterval = 5 * time.Second +) + +// containerState tracks the state of a container we've already processed. +type containerState struct { + appliedPIDs map[int]bool // GPU PIDs we've already applied controls to +} + +type containerInfo struct { + ID string + PID int + Name string + EnvVars map[string]string +} + +func main() { + fmt.Println("GVM Docker Daemon v2 starting...") + fmt.Printf(" Poll interval: %s\n", pollInterval) + fmt.Printf(" GVM sysfs path: %s\n", gvm.GVMProcessesPath) + + // Query total GPU memory for percentage-based limits + gpuMemory, err := gvm.QueryGPUTotalMemory() + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not query GPU memory: %v\n", err) + fmt.Fprintf(os.Stderr, " GVM_MEMORY_PERCENTAGE will not work.\n") + gpuMemory = make(map[int]int64) + } else { + for idx, mem := range gpuMemory { + fmt.Printf(" GPU %d: %s total memory\n", idx, gvm.FormatMemoryValue(mem)) + } + } + fmt.Println() + + // Track container states (supports detecting new GPU processes in already-known containers) + containerStates := make(map[string]*containerState) + + for { + containers, err := getRunningContainers() + if err != nil { + fmt.Fprintf(os.Stderr, "[%s] Error listing containers: %v\n", ts(), err) + time.Sleep(pollInterval) + continue + } + + // Track which container IDs are still running for cleanup + activeIDs := make(map[string]bool) + + for _, c := range containers { + activeIDs[c.ID] = true + + // Parse GVM config from container env vars + config, err := gvm.ConfigFromEnv(c.EnvVars) + if err != nil { + fmt.Fprintf(os.Stderr, "[%s] Error parsing config for %s: %v\n", ts(), c.Name, err) + continue + } + + // Skip if GVM is disabled or no controls set + if !config.Enabled || !config.HasAnyControl() { + continue + } + + // Initialize state tracking for this container + if _, ok := containerStates[c.ID]; !ok { + containerStates[c.ID] = &containerState{ + appliedPIDs: make(map[int]bool), + } + fmt.Printf("[%s] Discovered GVM container: %s (PID: %d)\n", ts(), c.Name, c.PID) + if config.Debug { + fmt.Printf("[%s] Memory limit: %s\n", ts(), describeMemoryConfig(config)) + fmt.Printf("[%s] Compute priority: %d\n", ts(), config.ComputePriority) + fmt.Printf("[%s] Compute freeze: %v\n", ts(), config.ComputeFreeze) + } + } + + state := containerStates[c.ID] + + // Find GPU processes for this container + gpuPIDs, err := gvm.FindGPUPIDs(c.PID) + if err != nil { + if config.Debug { + fmt.Fprintf(os.Stderr, "[%s] Error finding GPU PIDs for %s: %v\n", ts(), c.Name, err) + } + continue + } + + if len(gpuPIDs) == 0 { + continue + } + + // Apply controls to any new GPU PIDs + for _, gpuPID := range gpuPIDs { + if state.appliedPIDs[gpuPID] { + continue + } + + // Discover GPU indices for this PID + gpuIndices, err := gvm.ListGPUIndices(gpuPID) + if err != nil { + if config.Debug { + fmt.Fprintf(os.Stderr, "[%s] Error listing GPU indices for PID %d: %v\n", ts(), gpuPID, err) + } + continue + } + + allOK := true + for _, gpuIdx := range gpuIndices { + totalGPUMem := gpuMemory[gpuIdx] + + if err := gvm.ApplyConfig(gpuPID, gpuIdx, config, totalGPUMem); err != nil { + fmt.Fprintf(os.Stderr, "[%s] Error applying controls to PID %d GPU %d (container %s): %v\n", + ts(), gpuPID, gpuIdx, c.Name, err) + allOK = false + } else { + memLimit := config.MemoryLimitForDevice(gpuIdx, totalGPUMem) + fmt.Printf("[%s] Applied controls to PID %d GPU %d (container: %s) — memory=%s priority=%d freeze=%v\n", + ts(), gpuPID, gpuIdx, c.Name, + gvm.FormatMemoryValue(memLimit), config.ComputePriority, config.ComputeFreeze) + } + } + + if allOK { + state.appliedPIDs[gpuPID] = true + } + } + } + + // Cleanup stale containers + for id := range containerStates { + if !activeIDs[id] { + delete(containerStates, id) + } + } + + time.Sleep(pollInterval) + } +} + +func getRunningContainers() ([]containerInfo, error) { + cmd := exec.Command("docker", "ps", "-q") + output, err := cmd.Output() + if err != nil { + return nil, err + } + + raw := strings.TrimSpace(string(output)) + if raw == "" { + return nil, nil + } + + containerIDs := strings.Split(raw, "\n") + var containers []containerInfo + + for _, id := range containerIDs { + id = strings.TrimSpace(id) + if id == "" { + continue + } + + cmd := exec.Command("docker", "inspect", id) + output, err := cmd.Output() + if err != nil { + continue + } + + var inspectData []struct { + ID string `json:"Id"` + Name string `json:"Name"` + State struct { + Pid int `json:"Pid"` + } `json:"State"` + Config struct { + Env []string `json:"Env"` + } `json:"Config"` + } + + if err := json.Unmarshal(output, &inspectData); err != nil { + continue + } + if len(inspectData) == 0 { + continue + } + + data := inspectData[0] + + envVars := make(map[string]string) + for _, env := range data.Config.Env { + parts := strings.SplitN(env, "=", 2) + if len(parts) == 2 { + envVars[parts[0]] = parts[1] + } + } + + containers = append(containers, containerInfo{ + ID: data.ID, + PID: data.State.Pid, + Name: strings.TrimPrefix(data.Name, "/"), + EnvVars: envVars, + }) + } + + return containers, nil +} + +func ts() string { + return time.Now().Format("15:04:05") +} + +func describeMemoryConfig(config *gvm.Config) string { + parts := []string{} + if config.MemoryLimit > 0 { + parts = append(parts, fmt.Sprintf("global=%s", gvm.FormatMemoryValue(config.MemoryLimit))) + } + for idx, limit := range config.PerDeviceMemoryLimit { + parts = append(parts, fmt.Sprintf("gpu%d=%s", idx, gvm.FormatMemoryValue(limit))) + } + if config.MemoryPercentage > 0 { + parts = append(parts, fmt.Sprintf("percentage=%d%%", config.MemoryPercentage)) + } + if len(parts) == 0 { + return "unlimited" + } + return strings.Join(parts, ", ") +} diff --git a/gvm-docker/cmd/gvm-runc/main.go b/gvm-docker/cmd/gvm-runc/main.go new file mode 100644 index 00000000..9cbe8379 --- /dev/null +++ b/gvm-docker/cmd/gvm-runc/main.go @@ -0,0 +1,275 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/ovg-project/gvm-docker/pkg/gvm" +) + +var runcPath string + +func init() { + // Ensure we have root privileges + if syscall.Geteuid() != 0 { + fmt.Fprintln(os.Stderr, "gvm-runc: must run as root") + os.Exit(1) + } +} + +func main() { + // Special mode: apply GVM controls as a background process. + // Invoked as: gvm-runc --gvm-apply + if len(os.Args) >= 4 && os.Args[1] == "--gvm-apply" { + runApplyMode(os.Args[2], os.Args[3]) + return + } + + runcPath = findRunc() + + args := os.Args[1:] + + // Always log to file for debugging + logf := newLogger() + + logf("gvm-runc invoked: args=%v", args) + + // Parse the runc command, bundle path, and container ID from args. + // Docker typically calls: gvm-runc [global-flags] create --bundle + var command, bundlePath, containerID string + for i := 0; i < len(args); i++ { + arg := args[i] + if arg == "--bundle" && i+1 < len(args) { + bundlePath = args[i+1] + i++ // skip next + } else if arg == "create" || arg == "start" || arg == "run" { + command = arg + } else if !strings.HasPrefix(arg, "-") && command != "" { + // First non-flag arg after the command is the container ID + containerID = arg + } + } + + logf(" command=%s bundle=%s containerID=%s", command, bundlePath, containerID) + + // Read container env vars from the OCI bundle's config.json + var containerEnv map[string]string + if bundlePath != "" { + containerEnv = readBundleEnv(bundlePath, logf) + } + + // Execute the real runc + cmd := exec.Command(runcPath, args...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + os.Exit(exitErr.ExitCode()) + } + fmt.Fprintf(os.Stderr, "gvm-runc: failed to run runc: %v\n", err) + os.Exit(1) + } + + // After a successful create, fork a background process to apply GVM controls. + // We can't use a goroutine because main() returns and kills the goroutine. + if command == "create" && containerID != "" && len(containerEnv) > 0 { + envJSON, _ := json.Marshal(containerEnv) + child := exec.Command(os.Args[0], "--gvm-apply", containerID, string(envJSON)) + child.Stdout = nil + child.Stderr = nil + if err := child.Start(); err != nil { + logf("ERROR: failed to fork apply process: %v", err) + } else { + // Detach — don't wait for the child + child.Process.Release() + logf("Forked apply process (PID %d) for container %s", child.Process.Pid, containerID) + } + } +} + +func newLogger() func(string, ...interface{}) { + logFile, _ := os.OpenFile("/tmp/gvm-runc.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + return func(format string, a ...interface{}) { + msg := fmt.Sprintf("[%s] %s\n", time.Now().Format(time.RFC3339), fmt.Sprintf(format, a...)) + if logFile != nil { + fmt.Fprint(logFile, msg) + } + } +} + +// runApplyMode is the entry point for the background apply process. +func runApplyMode(containerID, envJSON string) { + logf := newLogger() + logf("Apply process started for container %s", containerID) + + var envVars map[string]string + if err := json.Unmarshal([]byte(envJSON), &envVars); err != nil { + logf("ERROR: failed to parse env JSON: %v", err) + return + } + + applyControls(containerID, envVars, logf) +} + +// readBundleEnv reads the container's environment variables from the OCI bundle config.json. +func readBundleEnv(bundlePath string, logf func(string, ...interface{})) map[string]string { + configPath := filepath.Join(bundlePath, "config.json") + data, err := os.ReadFile(configPath) + if err != nil { + logf("WARNING: could not read bundle config %s: %v", configPath, err) + return nil + } + + var ociConfig struct { + Process struct { + Env []string `json:"env"` + } `json:"process"` + } + if err := json.Unmarshal(data, &ociConfig); err != nil { + logf("WARNING: could not parse bundle config %s: %v", configPath, err) + return nil + } + + envVars := make(map[string]string) + for _, env := range ociConfig.Process.Env { + parts := strings.SplitN(env, "=", 2) + if len(parts) == 2 && strings.HasPrefix(parts[0], "GVM_") { + envVars[parts[0]] = parts[1] + } + } + + if len(envVars) > 0 { + logf(" Found GVM env vars in bundle: %v", envVars) + } + + return envVars +} + +func findRunc() string { + // Avoid finding ourselves — skip /usr/local/bin/gvm-runc + paths := []string{ + "/usr/bin/runc", + "/usr/sbin/runc", + "/run/current-system/sw/bin/runc", + } + for _, path := range paths { + if _, err := os.Stat(path); err == nil { + return path + } + } + if path, err := exec.LookPath("runc"); err == nil { + // Make sure we didn't find ourselves + if !strings.Contains(path, "gvm-runc") { + return path + } + } + return "/usr/bin/runc" +} + +func applyControls(containerID string, envVars map[string]string, logf func(string, ...interface{})) { + config, err := gvm.ConfigFromEnv(envVars) + if err != nil { + logf("ERROR parsing config for %s: %v", containerID, err) + return + } + + if !config.Enabled || !config.HasAnyControl() { + logf("No GVM controls to apply for container %s", containerID) + return + } + + logf("Starting GVM control application for container %s", containerID) + logf(" Memory limit: %s, Priority: %d, Freeze: %v", + gvm.FormatMemoryValue(config.MemoryLimit), config.ComputePriority, config.ComputeFreeze) + + // Get container PID via docker inspect + containerPID, err := getContainerPID(containerID) + if err != nil { + logf("ERROR: could not get container PID: %v", err) + return + } + logf("Container PID: %d", containerPID) + + // Retry finding GPU processes for up to 60 seconds + var pids []int + maxRetries := 30 + for i := 0; i < maxRetries; i++ { + time.Sleep(2 * time.Second) + + pids, err = gvm.FindGPUPIDs(containerPID) + if err != nil { + if i%5 == 0 { + logf("Attempt %d/%d: error finding GPU processes for %s: %v", i+1, maxRetries, containerID, err) + } + continue + } + + if len(pids) > 0 { + logf("Found %d GPU process(es) after %d seconds: %v", len(pids), (i+1)*2, pids) + break + } + + if i%5 == 0 { + logf("Still waiting for GPU processes (attempt %d/%d)", i+1, maxRetries) + } + } + + if len(pids) == 0 { + logf("WARNING: No GPU processes found after %d seconds for %s", maxRetries*2, containerID) + return + } + + // Query GPU memory for percentage-based limits + gpuMemory, memErr := gvm.QueryGPUTotalMemory() + if memErr != nil { + logf("Warning: could not query GPU memory: %v (percentage limits won't work)", memErr) + gpuMemory = make(map[int]int64) + } + + // Apply controls + for _, pid := range pids { + gpuIndices, err := gvm.ListGPUIndices(pid) + if err != nil { + logf("ERROR listing GPU indices for PID %d: %v", pid, err) + continue + } + + for _, gpuIdx := range gpuIndices { + totalMem := gpuMemory[gpuIdx] + if err := gvm.ApplyConfig(pid, gpuIdx, config, totalMem); err != nil { + logf("ERROR applying controls to PID %d GPU %d: %v", pid, gpuIdx, err) + } else { + memLimit := config.MemoryLimitForDevice(gpuIdx, totalMem) + logf("SUCCESS: Applied to PID %d GPU %d — memory=%s priority=%d freeze=%v", + pid, gpuIdx, gvm.FormatMemoryValue(memLimit), config.ComputePriority, config.ComputeFreeze) + } + } + } +} + +// getContainerPID uses docker inspect to get the main PID of a container. +func getContainerPID(containerID string) (int, error) { + cmd := exec.Command("docker", "inspect", "--format", "{{.State.Pid}}", containerID) + output, err := cmd.Output() + if err != nil { + return 0, fmt.Errorf("docker inspect failed: %w", err) + } + + pidStr := strings.TrimSpace(string(output)) + var pid int + if _, err := fmt.Sscanf(pidStr, "%d", &pid); err != nil { + return 0, fmt.Errorf("parse PID %q: %w", pidStr, err) + } + if pid == 0 { + return 0, fmt.Errorf("container not running") + } + return pid, nil +} diff --git a/gvm-docker/examples/diffusion/Dockerfile b/gvm-docker/examples/diffusion/Dockerfile new file mode 100644 index 00000000..00b68f90 --- /dev/null +++ b/gvm-docker/examples/diffusion/Dockerfile @@ -0,0 +1,48 @@ +FROM nvidia/cuda:12.0.0-base-ubuntu22.04 + +# Install Python and dependencies +RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + python3-venv \ + wget \ + gnupg2 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Install CUDA runtime libraries (needed for PyTorch) +RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb && \ + dpkg -i cuda-keyring_1.1-1_all.deb && \ + apt-get update && \ + apt-get install -y cuda-cudart-12-0 cuda-compat-12-0 && \ + rm -rf /var/lib/apt/lists/* cuda-keyring_1.1-1_all.deb + +# Create app directory +WORKDIR /app + +# Create virtual environment and install packages +RUN python3 -m venv /app/venv +ENV PATH="/app/venv/bin:$PATH" + +# Install PyTorch and diffusion dependencies +RUN pip3 install --no-cache-dir \ + torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121 && \ + pip3 install --no-cache-dir \ + diffusers \ + transformers \ + protobuf \ + sentencepiece \ + accelerate + +# Download diffusion script and prompts +RUN wget https://github.com/ovg-project/GVM/releases/download/v0.0.0-diffusion/diffusion.py && \ + wget https://github.com/ovg-project/GVM/releases/download/v0.0.0-diffusion/vidprom.txt + +# Create output directory +RUN mkdir -p /app/outputs + +# Set GVM intercept layer (will be mounted from host) +ENV LD_LIBRARY_PATH=/gvm-cuda-driver/install:$LD_LIBRARY_PATH + +# Default command +CMD ["python3", "diffusion.py", "--dataset_path=vidprom.txt", "--num_requests=10", "--log_file=outputs/stats.txt"] diff --git a/gvm-docker/examples/docker-compose/colocation.yml b/gvm-docker/examples/docker-compose/colocation.yml new file mode 100644 index 00000000..de6abf54 --- /dev/null +++ b/gvm-docker/examples/docker-compose/colocation.yml @@ -0,0 +1,55 @@ +# GPU workload colocation: high-priority inference + low-priority training. +# +# The inference server gets priority 2 (high) and 16GB memory. +# The training job gets priority 12 (low) and 8GB memory. +# GVM's kernel scheduler ensures inference latency is protected. +# +# Usage: +# docker compose -f colocation.yml up -d +# +# Prerequisites: +# - GVM kernel modules loaded +# - gvm-daemon running on host (sudo gvm-daemon) +# - NVIDIA Container Toolkit installed + +version: "3.8" + +services: + # High-priority inference server + inference: + image: nvidia/cuda:12.2.0-runtime-ubuntu22.04 + command: ["sleep", "infinity"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + GVM_MEMORY_LIMIT: "16g" + GVM_COMPUTE_PRIORITY: "2" + ports: + - "8000:8000" + + # Low-priority batch training + training: + image: nvidia/cuda:12.2.0-runtime-ubuntu22.04 + command: ["sleep", "infinity"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + GVM_MEMORY_LIMIT: "8g" + GVM_COMPUTE_PRIORITY: "12" + volumes: + - training-data:/data + - training-checkpoints:/checkpoints + +volumes: + training-data: + training-checkpoints: diff --git a/gvm-docker/examples/docker-compose/multi-gpu.yml b/gvm-docker/examples/docker-compose/multi-gpu.yml new file mode 100644 index 00000000..58a60b60 --- /dev/null +++ b/gvm-docker/examples/docker-compose/multi-gpu.yml @@ -0,0 +1,36 @@ +# Multi-GPU service with per-device memory limits. +# +# This example shows how to set different memory limits on each GPU +# for a distributed training workload. +# +# Usage: +# docker compose -f multi-gpu.yml up -d +# +# Prerequisites: +# - GVM kernel modules loaded +# - gvm-daemon running on host (sudo gvm-daemon) +# - NVIDIA Container Toolkit installed +# - At least 2 GPUs available + +version: "3.8" + +services: + distributed-training: + image: nvidia/cuda:12.2.0-runtime-ubuntu22.04 + command: ["sleep", "infinity"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 2 + capabilities: [gpu] + environment: + GVM_MEMORY_LIMIT_0: "20g" + GVM_MEMORY_LIMIT_1: "20g" + GVM_COMPUTE_PRIORITY: "5" + volumes: + - training-data:/data + +volumes: + training-data: diff --git a/gvm-docker/examples/docker-compose/percentage-limit.yml b/gvm-docker/examples/docker-compose/percentage-limit.yml new file mode 100644 index 00000000..ddae69b5 --- /dev/null +++ b/gvm-docker/examples/docker-compose/percentage-limit.yml @@ -0,0 +1,30 @@ +# Percentage-based memory limit example. +# +# Uses GVM_MEMORY_PERCENTAGE to set memory limit as a fraction of total GPU memory. +# This is portable across different GPU models (e.g., 50% of an A100 = 40GB, +# 50% of an L4 = 12GB). +# +# Usage: +# docker compose -f percentage-limit.yml up -d +# +# Prerequisites: +# - GVM kernel modules loaded +# - gvm-daemon running on host (sudo gvm-daemon) +# - NVIDIA Container Toolkit installed + +version: "3.8" + +services: + gpu-app: + image: nvidia/cuda:12.2.0-runtime-ubuntu22.04 + command: ["sleep", "infinity"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + GVM_MEMORY_PERCENTAGE: "50" + GVM_COMPUTE_PRIORITY: "8" diff --git a/gvm-docker/examples/docker-compose/single-service.yml b/gvm-docker/examples/docker-compose/single-service.yml new file mode 100644 index 00000000..561eed75 --- /dev/null +++ b/gvm-docker/examples/docker-compose/single-service.yml @@ -0,0 +1,26 @@ +# Single GPU service with GVM memory and priority controls. +# +# Usage: +# docker compose -f single-service.yml up -d +# +# Prerequisites: +# - GVM kernel modules loaded +# - gvm-daemon running on host (sudo gvm-daemon) +# - NVIDIA Container Toolkit installed + +version: "3.8" + +services: + gpu-app: + image: nvidia/cuda:12.2.0-runtime-ubuntu22.04 + command: ["nvidia-smi", "-l", "5"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + GVM_MEMORY_LIMIT: "6g" + GVM_COMPUTE_PRIORITY: "8" diff --git a/gvm-docker/examples/test-colocation.sh b/gvm-docker/examples/test-colocation.sh new file mode 100644 index 00000000..d207c75a --- /dev/null +++ b/gvm-docker/examples/test-colocation.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Test script for GVM-Docker colocation + +set -e + +echo "=== GVM-Docker Colocation Test ===" +echo "" + +# Check if Docker is configured with GVM runtime +if ! docker info 2>/dev/null | grep -q "gvm"; then + echo "ERROR: Docker not configured with GVM runtime" + echo "Please add GVM runtime to /etc/docker/daemon.json and restart Docker" + exit 1 +fi + +echo "✓ Docker configured with GVM runtime" +echo "" + +# Build diffusion image if needed +if ! docker images | grep -q "gvm-diffusion"; then + echo "Building diffusion image..." + cd examples/diffusion + docker build -t gvm-diffusion . + cd ../.. +fi + +echo "✓ Diffusion image ready" +echo "" + +# Clean up any existing containers +docker rm -f diffusion-test 2>/dev/null || true +docker rm -f vllm-test 2>/dev/null || true + +echo "=== Test 1: Single Container with GVM Controls ===" +echo "Starting diffusion with 6GB memory limit and priority 8..." +echo "" + +docker run -d \ + --runtime=gvm \ + --gpus all \ + --name diffusion-test \ + --env GVM_MEMORY_LIMIT=6000000000 \ + --env GVM_COMPUTE_PRIORITY=8 \ + --env GVM_DEBUG=true \ + -v ~/GVM/gvm-cuda-driver/install:/gvm-cuda-driver/install:ro \ + gvm-diffusion + +echo "Container started. Waiting for GPU process..." +sleep 10 + +# Get container PID +CONTAINER_PID=$(docker inspect -f '{{.State.Pid}}' diffusion-test) +echo "Container PID: $CONTAINER_PID" + +# Find GPU process +echo "" +echo "Checking GVM controls..." +for pid_dir in /sys/kernel/debug/nvidia-uvm/processes/*/; do + pid=$(basename "$pid_dir") + if [ -d "/proc/$pid" ]; then + # Check if this PID is in the container's process tree + if grep -q "^$CONTAINER_PID$" <(pstree -p "$pid" 2>/dev/null | grep -o '[0-9]\+') 2>/dev/null; then + echo "Found GPU process: $pid" + echo "" + echo "Memory limit:" + sudo cat "$pid_dir/0/memory.limit" + echo "" + echo "Compute priority:" + sudo cat "$pid_dir/0/compute.priority" + echo "" + echo "Current memory usage:" + sudo cat "$pid_dir/0/memory.current" + break + fi + fi +done + +echo "" +echo "Container logs:" +docker logs diffusion-test 2>&1 | head -20 + +echo "" +echo "Waiting 30 seconds for processing..." +sleep 30 + +echo "" +echo "Final status:" +docker logs diffusion-test 2>&1 | tail -10 + +# Cleanup +echo "" +echo "Cleaning up..." +docker rm -f diffusion-test + +echo "" +echo "=== Test Complete ===" diff --git a/gvm-docker/go.mod b/gvm-docker/go.mod new file mode 100644 index 00000000..b7c85bd4 --- /dev/null +++ b/gvm-docker/go.mod @@ -0,0 +1,3 @@ +module github.com/ovg-project/gvm-docker + +go 1.21 diff --git a/gvm-docker/pkg/gvm/config.go b/gvm-docker/pkg/gvm/config.go new file mode 100644 index 00000000..91a495f0 --- /dev/null +++ b/gvm-docker/pkg/gvm/config.go @@ -0,0 +1,243 @@ +package gvm + +import ( + "fmt" + "os" + "strconv" + "strings" +) + +// Config holds all GVM control parameters for a container. +type Config struct { + // Global memory limit in bytes. 0 means unlimited. + MemoryLimit int64 + + // Per-device memory limits in bytes, keyed by GPU index. + // Overrides MemoryLimit for the specified device. + PerDeviceMemoryLimit map[int]int64 + + // Memory limit as percentage of total GPU memory (1-100). + // Only used if MemoryLimit is not set. + MemoryPercentage int + + // Compute priority (0-15). 0 = highest priority, 15 = lowest. + ComputePriority int + + // Whether to freeze GPU compute for this container. + ComputeFreeze bool + + // Whether GVM controls are enabled for this container. + Enabled bool + + // Whether debug logging is enabled. + Debug bool +} + +const DefaultPriority = 8 + +// ConfigFromEnv parses GVM configuration from a map of environment variables. +// This is used by the daemon which reads env vars from docker inspect. +func ConfigFromEnv(envVars map[string]string) (*Config, error) { + config := &Config{ + ComputePriority: DefaultPriority, + Enabled: true, + PerDeviceMemoryLimit: make(map[int]int64), + } + + // GVM_ENABLED + if val, ok := envVars["GVM_ENABLED"]; ok { + config.Enabled = strings.ToLower(val) != "false" + } + + if !config.Enabled { + return config, nil + } + + // GVM_DEBUG + config.Debug = strings.ToLower(envVars["GVM_DEBUG"]) == "true" + + // GVM_MEMORY_LIMIT (global) + if val, ok := envVars["GVM_MEMORY_LIMIT"]; ok && val != "" { + bytes, err := ParseMemoryValue(val) + if err != nil { + return nil, fmt.Errorf("invalid GVM_MEMORY_LIMIT %q: %w", val, err) + } + config.MemoryLimit = bytes + } + + // GVM_MEMORY_LIMIT_ (per-device) + for key, val := range envVars { + if !strings.HasPrefix(key, "GVM_MEMORY_LIMIT_") { + continue + } + suffix := strings.TrimPrefix(key, "GVM_MEMORY_LIMIT_") + gpuIdx, err := strconv.Atoi(suffix) + if err != nil { + continue // not a numeric suffix, skip + } + bytes, err := ParseMemoryValue(val) + if err != nil { + return nil, fmt.Errorf("invalid %s %q: %w", key, val, err) + } + config.PerDeviceMemoryLimit[gpuIdx] = bytes + } + + // GVM_MEMORY_PERCENTAGE + if val, ok := envVars["GVM_MEMORY_PERCENTAGE"]; ok && val != "" { + pct, err := strconv.Atoi(val) + if err != nil || pct < 1 || pct > 100 { + return nil, fmt.Errorf("invalid GVM_MEMORY_PERCENTAGE %q: must be 1-100", val) + } + config.MemoryPercentage = pct + } + + // GVM_COMPUTE_PRIORITY + if val, ok := envVars["GVM_COMPUTE_PRIORITY"]; ok && val != "" { + p, err := strconv.Atoi(val) + if err != nil || p < 0 || p > 15 { + return nil, fmt.Errorf("invalid GVM_COMPUTE_PRIORITY %q: must be 0-15", val) + } + config.ComputePriority = p + } + + // GVM_COMPUTE_FREEZE + if val, ok := envVars["GVM_COMPUTE_FREEZE"]; ok { + config.ComputeFreeze = strings.ToLower(val) == "true" + } + + return config, nil +} + +// ConfigFromOSEnv parses GVM configuration from the current process environment. +// This is used by the runc wrapper which inherits container env vars. +func ConfigFromOSEnv() (*Config, error) { + envVars := make(map[string]string) + for _, env := range os.Environ() { + parts := strings.SplitN(env, "=", 2) + if len(parts) == 2 && strings.HasPrefix(parts[0], "GVM_") { + envVars[parts[0]] = parts[1] + } + } + return ConfigFromEnv(envVars) +} + +// HasAnyControl returns true if this config has any GVM control set +// (i.e., there's something to apply beyond defaults). +func (c *Config) HasAnyControl() bool { + if !c.Enabled { + return false + } + return c.MemoryLimit > 0 || + len(c.PerDeviceMemoryLimit) > 0 || + c.MemoryPercentage > 0 || + c.ComputePriority != DefaultPriority || + c.ComputeFreeze +} + +// MemoryLimitForDevice returns the effective memory limit in bytes for a +// specific GPU device index. Returns 0 if no limit is configured. +// +// Priority: per-device limit > global limit > percentage-based limit. +// For percentage-based, totalGPUMemory must be provided (in bytes). +func (c *Config) MemoryLimitForDevice(gpuIndex int, totalGPUMemory int64) int64 { + // Per-device override takes highest priority + if limit, ok := c.PerDeviceMemoryLimit[gpuIndex]; ok { + return limit + } + + // Global memory limit + if c.MemoryLimit > 0 { + return c.MemoryLimit + } + + // Percentage-based + if c.MemoryPercentage > 0 && totalGPUMemory > 0 { + return totalGPUMemory * int64(c.MemoryPercentage) / 100 + } + + return 0 +} + +// ParseMemoryValue parses a human-readable memory string into bytes. +// Supports: raw bytes ("6000000000"), or suffixed values ("6g", "6000m", "1500k", "100b"). +// Uses SI units (powers of 10): 1g = 1,000,000,000 bytes. +func ParseMemoryValue(s string) (int64, error) { + s = strings.TrimSpace(s) + if s == "" { + return 0, fmt.Errorf("empty memory value") + } + + // Check for unit suffix + lastChar := strings.ToLower(s[len(s)-1:]) + multiplier := int64(1) + numStr := s + + switch lastChar { + case "t": + multiplier = 1_000_000_000_000 + numStr = s[:len(s)-1] + case "g": + multiplier = 1_000_000_000 + numStr = s[:len(s)-1] + case "m": + multiplier = 1_000_000 + numStr = s[:len(s)-1] + case "k": + multiplier = 1_000 + numStr = s[:len(s)-1] + case "b": + multiplier = 1 + numStr = s[:len(s)-1] + default: + // No suffix — treat as raw bytes + multiplier = 1 + numStr = s + } + + numStr = strings.TrimSpace(numStr) + if numStr == "" { + return 0, fmt.Errorf("no numeric value in %q", s) + } + + // Support decimal values like "1.5g" + if strings.Contains(numStr, ".") { + val, err := strconv.ParseFloat(numStr, 64) + if err != nil { + return 0, fmt.Errorf("invalid number %q: %w", numStr, err) + } + if val < 0 { + return 0, fmt.Errorf("negative memory value: %s", s) + } + return int64(val * float64(multiplier)), nil + } + + val, err := strconv.ParseInt(numStr, 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid number %q: %w", numStr, err) + } + if val < 0 { + return 0, fmt.Errorf("negative memory value: %s", s) + } + + return val * multiplier, nil +} + +// FormatMemoryValue formats a byte count into a human-readable string. +func FormatMemoryValue(bytes int64) string { + if bytes == 0 { + return "0" + } + if bytes >= 1_000_000_000_000 && bytes%1_000_000_000_000 == 0 { + return fmt.Sprintf("%dt", bytes/1_000_000_000_000) + } + if bytes >= 1_000_000_000 && bytes%1_000_000_000 == 0 { + return fmt.Sprintf("%dg", bytes/1_000_000_000) + } + if bytes >= 1_000_000 && bytes%1_000_000 == 0 { + return fmt.Sprintf("%dm", bytes/1_000_000) + } + if bytes >= 1_000 && bytes%1_000 == 0 { + return fmt.Sprintf("%dk", bytes/1_000) + } + return fmt.Sprintf("%d", bytes) +} diff --git a/gvm-docker/pkg/gvm/config_test.go b/gvm-docker/pkg/gvm/config_test.go new file mode 100644 index 00000000..56261035 --- /dev/null +++ b/gvm-docker/pkg/gvm/config_test.go @@ -0,0 +1,286 @@ +package gvm + +import ( + "testing" +) + +func TestParseMemoryValue(t *testing.T) { + tests := []struct { + input string + expected int64 + wantErr bool + }{ + // Raw bytes + {"0", 0, false}, + {"1000", 1000, false}, + {"6000000000", 6000000000, false}, + + // Byte suffix + {"100b", 100, false}, + {"100B", 100, false}, + + // Kilobytes + {"1k", 1000, false}, + {"1K", 1000, false}, + {"1500k", 1500000, false}, + + // Megabytes + {"1m", 1000000, false}, + {"1M", 1000000, false}, + {"6000m", 6000000000, false}, + {"16000m", 16000000000, false}, + + // Gigabytes + {"1g", 1000000000, false}, + {"1G", 1000000000, false}, + {"6g", 6000000000, false}, + {"24g", 24000000000, false}, + + // Terabytes + {"1t", 1000000000000, false}, + {"1T", 1000000000000, false}, + + // Decimal values + {"1.5g", 1500000000, false}, + {"0.5g", 500000000, false}, + {"2.5m", 2500000, false}, + + // Whitespace + {" 6g ", 6000000000, false}, + {" 1000 ", 1000, false}, + + // Errors + {"", 0, true}, + {"g", 0, true}, + {"-1g", 0, true}, + {"abc", 0, true}, + {"abc-g", 0, true}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result, err := ParseMemoryValue(tt.input) + if tt.wantErr { + if err == nil { + t.Errorf("ParseMemoryValue(%q) expected error, got %d", tt.input, result) + } + return + } + if err != nil { + t.Errorf("ParseMemoryValue(%q) unexpected error: %v", tt.input, err) + return + } + if result != tt.expected { + t.Errorf("ParseMemoryValue(%q) = %d, want %d", tt.input, result, tt.expected) + } + }) + } +} + +func TestFormatMemoryValue(t *testing.T) { + tests := []struct { + input int64 + expected string + }{ + {0, "0"}, + {500, "500"}, + {1000, "1k"}, + {1500000, "1500k"}, + {1000000, "1m"}, + {6000000000, "6g"}, + {1000000000000, "1t"}, + {1500000000, "1500m"}, + {1234, "1234"}, + } + + for _, tt := range tests { + t.Run(tt.expected, func(t *testing.T) { + result := FormatMemoryValue(tt.input) + if result != tt.expected { + t.Errorf("FormatMemoryValue(%d) = %q, want %q", tt.input, result, tt.expected) + } + }) + } +} + +func TestConfigFromEnv(t *testing.T) { + t.Run("empty env", func(t *testing.T) { + config, err := ConfigFromEnv(map[string]string{}) + if err != nil { + t.Fatal(err) + } + if !config.Enabled { + t.Error("expected Enabled=true by default") + } + if config.ComputePriority != DefaultPriority { + t.Errorf("expected default priority %d, got %d", DefaultPriority, config.ComputePriority) + } + if config.HasAnyControl() { + t.Error("expected HasAnyControl()=false with no env vars") + } + }) + + t.Run("basic controls", func(t *testing.T) { + config, err := ConfigFromEnv(map[string]string{ + "GVM_MEMORY_LIMIT": "6g", + "GVM_COMPUTE_PRIORITY": "2", + }) + if err != nil { + t.Fatal(err) + } + if config.MemoryLimit != 6000000000 { + t.Errorf("MemoryLimit = %d, want 6000000000", config.MemoryLimit) + } + if config.ComputePriority != 2 { + t.Errorf("ComputePriority = %d, want 2", config.ComputePriority) + } + if !config.HasAnyControl() { + t.Error("expected HasAnyControl()=true") + } + }) + + t.Run("per-device limits", func(t *testing.T) { + config, err := ConfigFromEnv(map[string]string{ + "GVM_MEMORY_LIMIT": "4g", + "GVM_MEMORY_LIMIT_0": "8g", + "GVM_MEMORY_LIMIT_1": "6g", + }) + if err != nil { + t.Fatal(err) + } + if config.MemoryLimit != 4000000000 { + t.Errorf("MemoryLimit = %d, want 4000000000", config.MemoryLimit) + } + if config.PerDeviceMemoryLimit[0] != 8000000000 { + t.Errorf("PerDeviceMemoryLimit[0] = %d, want 8000000000", config.PerDeviceMemoryLimit[0]) + } + if config.PerDeviceMemoryLimit[1] != 6000000000 { + t.Errorf("PerDeviceMemoryLimit[1] = %d, want 6000000000", config.PerDeviceMemoryLimit[1]) + } + }) + + t.Run("percentage", func(t *testing.T) { + config, err := ConfigFromEnv(map[string]string{ + "GVM_MEMORY_PERCENTAGE": "50", + }) + if err != nil { + t.Fatal(err) + } + if config.MemoryPercentage != 50 { + t.Errorf("MemoryPercentage = %d, want 50", config.MemoryPercentage) + } + }) + + t.Run("freeze", func(t *testing.T) { + config, err := ConfigFromEnv(map[string]string{ + "GVM_COMPUTE_FREEZE": "true", + }) + if err != nil { + t.Fatal(err) + } + if !config.ComputeFreeze { + t.Error("expected ComputeFreeze=true") + } + }) + + t.Run("disabled", func(t *testing.T) { + config, err := ConfigFromEnv(map[string]string{ + "GVM_ENABLED": "false", + "GVM_MEMORY_LIMIT": "6g", + }) + if err != nil { + t.Fatal(err) + } + if config.Enabled { + t.Error("expected Enabled=false") + } + // When disabled, memory limit should not be parsed + if config.MemoryLimit != 0 { + t.Errorf("MemoryLimit = %d, want 0 (disabled)", config.MemoryLimit) + } + }) + + t.Run("debug", func(t *testing.T) { + config, err := ConfigFromEnv(map[string]string{ + "GVM_DEBUG": "true", + }) + if err != nil { + t.Fatal(err) + } + if !config.Debug { + t.Error("expected Debug=true") + } + }) + + t.Run("invalid priority", func(t *testing.T) { + _, err := ConfigFromEnv(map[string]string{ + "GVM_COMPUTE_PRIORITY": "20", + }) + if err == nil { + t.Error("expected error for priority 20") + } + }) + + t.Run("invalid percentage", func(t *testing.T) { + _, err := ConfigFromEnv(map[string]string{ + "GVM_MEMORY_PERCENTAGE": "150", + }) + if err == nil { + t.Error("expected error for percentage 150") + } + }) + + t.Run("invalid memory value", func(t *testing.T) { + _, err := ConfigFromEnv(map[string]string{ + "GVM_MEMORY_LIMIT": "notanumber", + }) + if err == nil { + t.Error("expected error for invalid memory value") + } + }) +} + +func TestMemoryLimitForDevice(t *testing.T) { + config := &Config{ + MemoryLimit: 4000000000, + PerDeviceMemoryLimit: map[int]int64{ + 0: 8000000000, + }, + MemoryPercentage: 50, + } + + t.Run("per-device override", func(t *testing.T) { + limit := config.MemoryLimitForDevice(0, 24000000000) + if limit != 8000000000 { + t.Errorf("GPU 0 limit = %d, want 8000000000 (per-device)", limit) + } + }) + + t.Run("global fallback", func(t *testing.T) { + limit := config.MemoryLimitForDevice(1, 24000000000) + if limit != 4000000000 { + t.Errorf("GPU 1 limit = %d, want 4000000000 (global)", limit) + } + }) + + t.Run("percentage fallback", func(t *testing.T) { + pctConfig := &Config{ + MemoryPercentage: 50, + PerDeviceMemoryLimit: make(map[int]int64), + } + limit := pctConfig.MemoryLimitForDevice(0, 24000000000) + if limit != 12000000000 { + t.Errorf("GPU 0 pct limit = %d, want 12000000000 (50%% of 24g)", limit) + } + }) + + t.Run("no limit", func(t *testing.T) { + noConfig := &Config{ + PerDeviceMemoryLimit: make(map[int]int64), + } + limit := noConfig.MemoryLimitForDevice(0, 24000000000) + if limit != 0 { + t.Errorf("GPU 0 limit = %d, want 0 (unlimited)", limit) + } + }) +} diff --git a/gvm-docker/pkg/gvm/process.go b/gvm-docker/pkg/gvm/process.go new file mode 100644 index 00000000..ced9c659 --- /dev/null +++ b/gvm-docker/pkg/gvm/process.go @@ -0,0 +1,121 @@ +package gvm + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +// FindGPUPIDs returns the list of PIDs that are GPU processes belonging to the +// given container PID (i.e., the container's init process or any of its descendants). +func FindGPUPIDs(containerPID int) ([]int, error) { + // Build the full PID tree for this container + allPIDs := []int{containerPID} + children := getChildProcesses(containerPID) + allPIDs = append(allPIDs, children...) + + // Read the nvidia-uvm processes directory + entries, err := os.ReadDir(GVMProcessesPath) + if err != nil { + return nil, fmt.Errorf("read %s: %w", GVMProcessesPath, err) + } + + // Build a set for O(1) lookup + pidSet := make(map[int]bool, len(allPIDs)) + for _, p := range allPIDs { + pidSet[p] = true + } + + // Match GPU PIDs against container PID tree + var gpuPIDs []int + for _, entry := range entries { + if !entry.IsDir() { + continue + } + pid, err := strconv.Atoi(entry.Name()) + if err != nil { + continue + } + if pidSet[pid] { + gpuPIDs = append(gpuPIDs, pid) + } + } + + return gpuPIDs, nil +} + +// ListAllGPUPIDs returns all PIDs that currently have GPU processes. +func ListAllGPUPIDs() ([]int, error) { + entries, err := os.ReadDir(GVMProcessesPath) + if err != nil { + return nil, fmt.Errorf("read %s: %w", GVMProcessesPath, err) + } + + var pids []int + for _, entry := range entries { + if !entry.IsDir() { + continue + } + pid, err := strconv.Atoi(entry.Name()) + if err != nil { + continue + } + pids = append(pids, pid) + } + return pids, nil +} + +// getChildProcesses recursively finds all descendant PIDs of the given PID +// by scanning /proc/*/stat for parent PID matches. +func getChildProcesses(pid int) []int { + var children []int + + entries, err := os.ReadDir("/proc") + if err != nil { + return nil + } + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + childPID, err := strconv.Atoi(entry.Name()) + if err != nil { + continue + } + + statPath := filepath.Join("/proc", entry.Name(), "stat") + data, err := os.ReadFile(statPath) + if err != nil { + continue + } + + // Parse stat file: pid (comm) state ppid ... + // The comm field can contain spaces and parentheses, so find the last ')' first. + content := string(data) + lastParen := strings.LastIndex(content, ")") + if lastParen == -1 || lastParen+2 >= len(content) { + continue + } + fieldsAfterComm := strings.Fields(content[lastParen+2:]) + if len(fieldsAfterComm) < 2 { + continue + } + // fieldsAfterComm[0] = state, fieldsAfterComm[1] = ppid + ppid, err := strconv.Atoi(fieldsAfterComm[1]) + if err != nil { + continue + } + + if ppid == pid { + children = append(children, childPID) + grandchildren := getChildProcesses(childPID) + children = append(children, grandchildren...) + } + } + + return children +} diff --git a/gvm-docker/pkg/gvm/sysfs.go b/gvm-docker/pkg/gvm/sysfs.go new file mode 100644 index 00000000..3a990939 --- /dev/null +++ b/gvm-docker/pkg/gvm/sysfs.go @@ -0,0 +1,212 @@ +package gvm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" +) + +const ( + // GVMProcessesPath is the sysfs base path for GVM GPU process controls. + GVMProcessesPath = "/sys/kernel/debug/nvidia-uvm/processes" +) + +// GPUProcessPath returns the sysfs base path for a specific PID and GPU index. +// e.g., /sys/kernel/debug/nvidia-uvm/processes/12345/0/ +func GPUProcessPath(pid int, gpuIndex int) string { + return filepath.Join(GVMProcessesPath, strconv.Itoa(pid), strconv.Itoa(gpuIndex)) +} + +// WriteSysfs writes a value to a sysfs file. Requires root privileges. +func WriteSysfs(path, value string) error { + file, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) + } + defer file.Close() + + _, err = file.WriteString(value) + if err != nil { + return fmt.Errorf("write %s to %s: %w", value, path, err) + } + return nil +} + +// ReadSysfs reads and trims the content of a sysfs file. +func ReadSysfs(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read %s: %w", path, err) + } + return strings.TrimSpace(string(data)), nil +} + +// SetMemoryLimit writes the memory limit (in bytes) for a GPU process. +func SetMemoryLimit(pid int, gpuIndex int, limitBytes int64) error { + path := filepath.Join(GPUProcessPath(pid, gpuIndex), "memory.limit") + return WriteSysfs(path, strconv.FormatInt(limitBytes, 10)) +} + +// GetMemoryLimit reads the current memory limit for a GPU process. +func GetMemoryLimit(pid int, gpuIndex int) (int64, error) { + path := filepath.Join(GPUProcessPath(pid, gpuIndex), "memory.limit") + val, err := ReadSysfs(path) + if err != nil { + return 0, err + } + return strconv.ParseInt(val, 10, 64) +} + +// GetMemoryCurrent reads the current GPU memory usage for a process. +func GetMemoryCurrent(pid int, gpuIndex int) (int64, error) { + path := filepath.Join(GPUProcessPath(pid, gpuIndex), "memory.current") + val, err := ReadSysfs(path) + if err != nil { + return 0, err + } + return strconv.ParseInt(val, 10, 64) +} + +// GetMemorySwapCurrent reads the current swapped memory for a GPU process. +func GetMemorySwapCurrent(pid int, gpuIndex int) (int64, error) { + path := filepath.Join(GPUProcessPath(pid, gpuIndex), "memory.swap.current") + val, err := ReadSysfs(path) + if err != nil { + return 0, err + } + return strconv.ParseInt(val, 10, 64) +} + +// SetComputePriority writes the compute priority (0-15) for a GPU process. +func SetComputePriority(pid int, gpuIndex int, priority int) error { + if priority < 0 || priority > 15 { + return fmt.Errorf("priority must be 0-15, got %d", priority) + } + path := filepath.Join(GPUProcessPath(pid, gpuIndex), "compute.priority") + return WriteSysfs(path, strconv.Itoa(priority)) +} + +// GetComputePriority reads the current compute priority for a GPU process. +func GetComputePriority(pid int, gpuIndex int) (int, error) { + path := filepath.Join(GPUProcessPath(pid, gpuIndex), "compute.priority") + val, err := ReadSysfs(path) + if err != nil { + return 0, err + } + return strconv.Atoi(val) +} + +// SetComputeFreeze freezes (true) or unfreezes (false) GPU compute for a process. +func SetComputeFreeze(pid int, gpuIndex int, freeze bool) error { + path := filepath.Join(GPUProcessPath(pid, gpuIndex), "compute.freeze") + val := "0" + if freeze { + val = "1" + } + return WriteSysfs(path, val) +} + +// GetComputeFreeze reads whether GPU compute is frozen for a process. +func GetComputeFreeze(pid int, gpuIndex int) (bool, error) { + path := filepath.Join(GPUProcessPath(pid, gpuIndex), "compute.freeze") + val, err := ReadSysfs(path) + if err != nil { + return false, err + } + return val == "1", nil +} + +// GetGCGroupStat reads the gcgroup stats for a GPU process. +func GetGCGroupStat(pid int, gpuIndex int) (string, error) { + path := filepath.Join(GPUProcessPath(pid, gpuIndex), "gcgroup.stat") + return ReadSysfs(path) +} + +// ListGPUIndices returns the GPU indices available for a given PID. +func ListGPUIndices(pid int) ([]int, error) { + pidPath := filepath.Join(GVMProcessesPath, strconv.Itoa(pid)) + entries, err := os.ReadDir(pidPath) + if err != nil { + return nil, fmt.Errorf("read %s: %w", pidPath, err) + } + + var indices []int + for _, entry := range entries { + if !entry.IsDir() { + continue + } + idx, err := strconv.Atoi(entry.Name()) + if err != nil { + continue + } + indices = append(indices, idx) + } + return indices, nil +} + +// QueryGPUTotalMemory returns a map of GPU index → total memory in bytes. +// Uses nvidia-smi to query device memory. Returns an error if nvidia-smi is unavailable. +func QueryGPUTotalMemory() (map[int]int64, error) { + cmd := exec.Command("nvidia-smi", "--query-gpu=index,memory.total", "--format=csv,noheader,nounits") + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("nvidia-smi query failed: %w", err) + } + + result := make(map[int]int64) + for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.SplitN(line, ",", 2) + if len(parts) != 2 { + continue + } + idx, err := strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil { + continue + } + // nvidia-smi reports memory in MiB + mib, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64) + if err != nil { + continue + } + result[idx] = mib * 1024 * 1024 // Convert MiB to bytes + } + + return result, nil +} + +// ApplyConfig applies a full GVM Config to a specific GPU process. +// totalGPUMemory is needed for percentage-based limits (pass 0 if unknown). +func ApplyConfig(pid int, gpuIndex int, config *Config, totalGPUMemory int64) error { + if !config.Enabled { + return nil + } + + // Apply memory limit + memLimit := config.MemoryLimitForDevice(gpuIndex, totalGPUMemory) + if memLimit > 0 { + if err := SetMemoryLimit(pid, gpuIndex, memLimit); err != nil { + return fmt.Errorf("set memory.limit for PID %d GPU %d: %w", pid, gpuIndex, err) + } + } + + // Apply compute priority + if err := SetComputePriority(pid, gpuIndex, config.ComputePriority); err != nil { + return fmt.Errorf("set compute.priority for PID %d GPU %d: %w", pid, gpuIndex, err) + } + + // Apply compute freeze + if config.ComputeFreeze { + if err := SetComputeFreeze(pid, gpuIndex, true); err != nil { + return fmt.Errorf("set compute.freeze for PID %d GPU %d: %w", pid, gpuIndex, err) + } + } + + return nil +} diff --git a/setup b/setup index b01e5f29..dd0d3d91 100755 --- a/setup +++ b/setup @@ -34,6 +34,8 @@ case "$1" in wget https://github.com/ovg-project/GVM/releases/download/v0.0.0-diffusion/vidprom.txt popd deactivate + # Create output directory for logs + mkdir -p diffusion_outputs/diffusion ;; llamafactory) echo "Running llamafactory logic..."