From ceba96df377a8367454a7f759d0e01c39d3dc713 Mon Sep 17 00:00:00 2001 From: khluu Date: Thu, 28 May 2026 17:33:55 -0700 Subject: [PATCH 1/2] Install pytest in container to fix cupy/torch.compile import failure Some vLLM release images don't include pytest, but cupy.testing._random imports it unconditionally. When torch.compile traces through cupy modules during model loading, this causes a ModuleNotFoundError that crashes the server. Installing pytest after container startup fixes this. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/server.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/server.sh b/lib/server.sh index c80fdbf..2d8ce49 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -55,6 +55,9 @@ start_server() { "$image" \ "$model" --port "$port" $serve_args + # Install pytest to avoid cupy.testing import failure during torch.compile + docker exec "$container" pip install -q pytest 2>/dev/null || true + echo "--- :memo: streaming vllm logs" ( docker logs -f "$container" 2>&1 | stdbuf -oL -eL sed 's/^/[vllm] /' ) & VLLM_LOGS_PID=$! From ff6f436c2e083872c5a02463aae8418181604896 Mon Sep 17 00:00:00 2001 From: khluu Date: Mon, 15 Jun 2026 13:36:41 -0700 Subject: [PATCH 2/2] Reap stale perf-eval containers before starting vLLM A job killed with SIGKILL (e.g. the Buildkite 120-min step timeout or an agent restart) never runs run.sh's EXIT trap, so its vLLM container survives and squats on the GPUs and port 8000. Because container names are PID-based and unique, a later run's own cleanup never reaps the orphan, and `docker run -p 8000:8000` then fails with "Bind for 0.0.0.0:8000 failed: port is already allocated" -- wedging every subsequent run on that host until someone removes the container by hand. This is what broke build 222 on h200-ci-1. Add reap_stale_servers(), called from start_server before `docker run`, to remove any leftover perf-eval container (by name prefix) or anything still holding the target port. The server port is fixed, so two perf-eval runs can't share a host -- any pre-existing perf-eval container is stale and safe to remove, making the host self-heal on the next run. This change was authored with assistance from Claude Code (AI-assisted). Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/server.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/server.sh b/lib/server.sh index 2d8ce49..5e8746c 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -35,6 +35,8 @@ start_server() { return fi + reap_stale_servers "$port" + local docker_args=(--gpus all --ipc=host --ulimit nofile=65536:65536 -e VLLM_ENGINE_READY_TIMEOUT_S=3600 -p "${port}:${port}") @@ -92,6 +94,26 @@ wait_healthy() { return 1 } +# Remove any leftover perf-eval vLLM container before starting a new one. +# A job killed with SIGKILL (e.g. the Buildkite step timeout) never runs its +# EXIT trap, so its container survives and squats on the GPUs and port 8000, +# wedging every later run on the host with "Bind for 0.0.0.0:8000 failed: port +# is already allocated". Container names are PID-based and unique, so a normal +# run's own cleanup never reaps a previous run's orphan. The server port is +# fixed, so two perf-eval runs can't share a host -- any existing perf-eval +# container is therefore stale and safe to remove. +reap_stale_servers() { + local port=$1 stale on_port + stale=$(docker ps -aq --filter "name=perf-eval-" 2>/dev/null || true) + on_port=$(docker ps -q --filter "publish=${port}" 2>/dev/null || true) + stale=$(printf '%s\n%s\n' "$stale" "$on_port" | sort -u | grep -v '^$' || true) + if [[ -n "$stale" ]]; then + echo "--- :broom: removing stale container(s) holding GPUs/port ${port}" + # shellcheck disable=SC2086 # word-split intended: one or more container ids + docker rm -f $stale >/dev/null 2>&1 || true + fi +} + stop_server() { local container=$1 if [[ -n "${VLLM_LOGS_PID:-}" ]]; then