diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index 9dc9648..25c02fd 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -57,6 +57,7 @@ def setup_command(packages): GPU_EMOJI = { "H200": ":h200:", "B200": ":b200:", + "GB300_SLURM": ":gb300:", "A100": ":a100:", "MI355X": ":amd:", "MI300X": ":amd:", diff --git a/README.md b/README.md index f01a6e6..b2eeaf5 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,10 @@ Include one or more of `lm_eval:` / `vllm_bench:` / `bfcl:` depending on what yo name: qwen3_5-h200 # used in container name and results// gpu: H200 # picks queue/image/HF cache from lib/gpu_profiles.yaml num_gpus: 8 +num_nodes: 1 # optional; set >1 for multi-node Slurm workloads +gpus_per_node: 8 # optional; used for Slurm allocation + metadata nightly: true # include in the nightly schedule (default: false) +bench_only: false # optional; true skips lm_eval/BFCL for this workload vllm: # how the server is brought up model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -73,6 +76,12 @@ bfcl: # function-calling eval (optional) multi_turn: 100 # or set a single int to cap every category vllm_bench: # perf runs (optional) — fed to the perf dashboard + metadata: # optional dashboard labels / throughput denominator + device: h200 + precision: fp8 + is_multinode: false + gpu_count: 8 + tp: 8 configs: - name: 1k-in-1k-out-conc-256 backend: openai # /v1/completions — exact ISL/OSL, no chat template @@ -86,6 +95,8 @@ vllm_bench: # perf runs (optional) — fed to the perf dashboard A few things worth knowing: - **`gpu`** must match a key in `lib/gpu_profiles.yaml`. The profile sets the Buildkite queue, default image, HF cache path, and baseline env vars. +- **`num_nodes`** and **`gpus_per_node`** describe the allocation shape for multi-node workloads. Existing single-node recipes can omit them. +- **`bench_only: true`** skips lm-eval and BFCL for one workload, even when the build-wide `BENCH_ONLY` env var is not set. This is useful for new perf-only smoke tests. - **`nightly`** controls only the nightly schedule. Recipes with `nightly: false` (or omitted) are still triggerable explicitly via the `WORKLOADS` env var. - **`lm_eval.tasks` is a list** because each entry runs as a separate `lm_eval` invocation — `--num_fewshot` is a single global flag, so different shot counts need separate runs. Each task's results land in `results///`. - **`vllm_bench` runs first** if both blocks are present — that way perf-pipeline bugs surface quickly instead of waiting on a full lm-eval pass. @@ -93,6 +104,8 @@ A few things worth knowing: - **`bfcl` may need tool-call serve args.** Some models require `--enable-auto-tool-choice` and `--tool-call-parser` for function-calling; the parser warns if `--tool-call-parser` is absent. Each category runs as a separate generate + evaluate pass; scores appear on the eval dashboard as `bfcl_` tasks. - **`bfcl.maximum_step_limit`** caps how many inference steps BFCL allows per multi-turn turn (default 10 in perf-eval; BFCL upstream defaults to 20). Set it in the workload YAML, or override per-run with the `BFCL_MAXIMUM_STEP_LIMIT` env var (env wins over YAML). Useful for agentic / long multi-turn categories. - **`bfcl.max_test_cases`** subsamples a category instead of running the full set — e.g. `multi_turn` (~800 cases) down to 300. For aggregate groups with multiple subcategories, the cap is split evenly across subcategories (by BFCL id order within each). Set a single integer to cap every category, or a map per category (`multi_turn: 240`). Override per-run with `BFCL_MAX_TEST_CASES`. Scores are partial-eval only and are not comparable to full BFCL leaderboard numbers. +- **`vllm_bench.metadata.gpu_count`** is the denominator for per-GPU throughput. Set it explicitly for multi-node runs; otherwise the parser falls back to `num_gpus` or the effective TP/DP parsed from `serve_args`. +- **`vllm_bench.metadata`** can also set dashboard fields such as `is_multinode`, `num_nodes`, `gpus_per_node`, `tp`, `dp`, `ep`, `dp_attention`, `disagg`, and `spec_decoding`. For everything else (the full set of supported fields, defaults, validation rules), the existing files in `workloads/` are the working reference and `lib/parse_workload.py` is the source of truth. @@ -126,6 +139,37 @@ The pipeline is [**`vllm/perf-eval`**](https://buildkite.com/vllm/perf-eval). Wi This runs the `qwen3_5_h200` workload against the specified vLLM nightly image. Omit `WORKLOADS` to run all `nightly: true` workloads. +### Run a GB300 Slurm smoke + +`workloads/deepseek_v4_flash_gb300_slurm.yaml` is an opt-in multi-node smoke. It does not run in nightly by default. Trigger it explicitly after the `GB300_SLURM` profile's Buildkite queue and Slurm launcher environment are configured: + +```bash +WORKLOADS=deepseek_v4_flash_gb300_slurm +BENCH_ONLY=1 +VLLM_COMMIT= +VLLM_IMAGE=/home/inf-simon/containers/vllm-openai-v0.20.0-cu130.sqsh +``` + +The `GB300_SLURM` profile uses `server_runtime: slurm`, which delegates server startup to `lib/slurm_launch_vllm.sh` unless `PERF_EVAL_SLURM_LAUNCHER` points at a custom launcher. The launcher contract is: + +1. Submit a long-running server job. +2. Write the endpoint env file named by `PERF_EVAL_ENDPOINT_FILE`. +3. Print the Slurm job id on stdout so `lib/server.sh` can `scancel` it on exit. + +Useful Slurm env vars: + +- `PERF_EVAL_SLURM_ACCOUNT`, `PERF_EVAL_SLURM_PARTITION`, `PERF_EVAL_SLURM_QOS`, `PERF_EVAL_SLURM_RESERVATION`, `PERF_EVAL_SLURM_TIME` +- `PERF_EVAL_SLURM_CONTAINER_MOUNTS`, `PERF_EVAL_SLURM_CONTAINER_WORKDIR`, `PERF_EVAL_SLURM_EXTRA_SBATCH_ARGS`, `PERF_EVAL_SLURM_EXTRA_SRUN_ARGS` +- `PERF_EVAL_SLURM_CONTAINER_RUNTIME=auto|pyxis|none` to choose Pyxis/Enroot container launch or native `srun`; `auto` uses Pyxis only when `srun --help` advertises `--container-image` +- `PERF_EVAL_SLURM_REQUEST_GPUS=0` for clusters that allocate whole GPU nodes without Slurm GPU GRES, `PERF_EVAL_SLURM_GPUS_PER_NODE=` to override the workload's metadata value for `--gpus-per-node`, or `PERF_EVAL_SLURM_GRES=` to request GPUs with `--gres` instead +- `PERF_EVAL_SLURM_VLLM_DISTRIBUTED_BACKEND=mp|none` to control the default native vLLM multi-node launcher. The default is `mp` when `num_nodes > 1`. +- `PERF_EVAL_SLURM_SERVER_COMMAND` to replace the default non-Ray `vllm serve` command with a cluster-specific launch wrapper +- `PERF_EVAL_BENCH_CLIENT_RUNTIME=slurm` to run `vllm bench serve` through Slurm instead of requiring a local `vllm` CLI on the Buildkite submission host; `PERF_EVAL_BENCH_CLIENT_CONTAINER_RUNTIME=auto|pyxis|none` overrides the server container runtime for that client step + +The GB300 Slurm profile defaults to Pyxis/Enroot containers for both the server and benchmark client, partition `batch`, `PERF_EVAL_SLURM_GRES=gpu:b300:4`, a 3-hour time limit, `--exclusive`, and `HF_HOME=/local_scratch/hf-models`. The profile starts with the Axis mounts `/home/inf-simon:/nfs_home,/raid/users/inf-simon:/local_scratch`; add a CI model-cache mount only after the durable cache path exists on the rack. If `PERF_EVAL_SLURM_GRES` is unset, the launcher uses the workload's `gpus_per_node` value with `--gpus-per-node`. + +For `num_nodes > 1`, the default Slurm launcher uses native vLLM MP serving rather than Ray: it starts one `vllm serve` task per node with `--distributed-executor-backend mp`, sets `--nnodes`, `--master-addr`, and `--node-rank`, and adds `--headless` on follower nodes. Pyxis server and benchmark-client steps set `--container-workdir` automatically when the current host checkout is under one of the configured `host:container` mounts; set `PERF_EVAL_SLURM_CONTAINER_WORKDIR` explicitly if the checkout path is not inferable. SRT-Slurm remains a possible future wrapper, but it needs an SRT/Dynamo-capable image and checkout and should not block this substrate path. + **From an agent:** see `CLAUDE.md` for the Buildkite MCP workflow (don't shell out to `curl` or `bk`). ### Run a recipe end-to-end diff --git a/lib/gpu_profiles.yaml b/lib/gpu_profiles.yaml index 63429b4..f86f636 100644 --- a/lib/gpu_profiles.yaml +++ b/lib/gpu_profiles.yaml @@ -15,6 +15,22 @@ B200: VLLM_DEEP_GEMM_WARMUP: skip NCCL_CUMEM_HOST_ENABLE: 0 +GB300_SLURM: + queue: gb300_slurm + hf_home: /local_scratch/hf-models + server_runtime: slurm + env: + VLLM_DEEP_GEMM_WARMUP: skip + PERF_EVAL_SLURM_CONTAINER_RUNTIME: pyxis + PERF_EVAL_BENCH_CLIENT_RUNTIME: slurm + PERF_EVAL_BENCH_CLIENT_CONTAINER_RUNTIME: pyxis + PERF_EVAL_SLURM_REQUEST_GPUS: "1" + PERF_EVAL_SLURM_PARTITION: batch + PERF_EVAL_SLURM_GRES: gpu:b300:4 + PERF_EVAL_SLURM_TIME: "03:00:00" + PERF_EVAL_SLURM_CONTAINER_MOUNTS: /home/inf-simon:/nfs_home,/raid/users/inf-simon:/local_scratch + PERF_EVAL_SLURM_EXTRA_SBATCH_ARGS: --exclusive + MI300X: queue: mi300_perf_eval image_repo: vllm/vllm-openai-rocm diff --git a/lib/ingest_perf.py b/lib/ingest_perf.py index 5291d43..236a61c 100644 --- a/lib/ingest_perf.py +++ b/lib/ingest_perf.py @@ -8,8 +8,8 @@ Latencies in the raw result are in milliseconds (e.g., `mean_ttft_ms`). The dashboard expects seconds, so the transform drops the `_ms` suffix and -divides by 1000. Aggregate throughput is divided by `tp` to match the -dashboard's per-GPU columns (`tput_per_gpu`, `output_tput_per_gpu`, +divides by 1000. Aggregate throughput is divided by `gpu_count` to populate +the dashboard's per-GPU columns (`tput_per_gpu`, `output_tput_per_gpu`, `input_tput_per_gpu`). Failures are logged but never fatal: ingestion is best-effort and must not @@ -41,9 +41,14 @@ def post(endpoint: str, payload: dict) -> None: raise RuntimeError(f"HTTP {resp.status}") +def bool_text(value: object) -> str: + return "true" if str(value).lower() in {"1", "true", "yes"} else "false" + + def transform(raw: dict, args: argparse.Namespace) -> dict: """Map the raw `vllm bench serve` JSON to the dashboard's row shape.""" tp = max(args.tp, 1) + gpu_count = max(args.gpu_count or tp, 1) total_token_throughput = float(raw.get("total_token_throughput", 0) or 0) output_throughput = float(raw.get("output_throughput", 0) or 0) input_throughput = total_token_throughput - output_throughput @@ -56,18 +61,25 @@ def transform(raw: dict, args: argparse.Namespace) -> dict: "model": raw.get("model_id") or args.model, "framework": "vllm", "precision": args.precision, - "spec_decoding": "false", - "disagg": "false", "isl": int(args.isl), "osl": int(args.osl), - "is_multinode": "false", + "is_multinode": bool_text(args.is_multinode), "tp": tp, - "ep": 1, - "dp_attention": "false", - "tput_per_gpu": total_token_throughput / tp, - "output_tput_per_gpu": output_throughput / tp, - "input_tput_per_gpu": input_throughput / tp, + "ep": args.ep or 1, + "dp_attention": bool_text(args.dp_attention), + "disagg": bool_text(args.disagg), + "spec_decoding": bool_text(args.spec_decoding), + "tput_per_gpu": total_token_throughput / gpu_count, + "output_tput_per_gpu": output_throughput / gpu_count, + "input_tput_per_gpu": input_throughput / gpu_count, } + if args.dp is not None: + data["dp"] = args.dp + if args.num_nodes is not None: + data["num_nodes"] = args.num_nodes + if args.gpus_per_node is not None: + data["gpus_per_node"] = args.gpus_per_node + data["gpu_count"] = gpu_count if os.environ.get("NIGHTLY") == "1": data["nightly"] = True @@ -89,11 +101,26 @@ def transform(raw: dict, args: argparse.Namespace) -> dict: return data +def optional_int(value: str) -> int | None: + if value in ("", "None", "none", "null"): + return None + return int(value) + + def main() -> int: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--raw-result", required=True, help="Raw JSON from `vllm bench serve --save-result`") p.add_argument("--device", required=True, help="Device tag (e.g. h200)") p.add_argument("--tp", type=int, required=True, help="Effective parallel-degree (TP * DP)") + p.add_argument("--dp", type=optional_int, default=None, help="Data parallel degree, when known") + p.add_argument("--ep", type=optional_int, default=None, help="Expert parallel degree, when known") + p.add_argument("--gpu-count", type=optional_int, default=None, help="Denominator for per-GPU throughput") + p.add_argument("--is-multinode", default="false", help="Whether this row came from a multi-node run") + p.add_argument("--num-nodes", type=optional_int, default=None, help="Node count for multi-node runs") + p.add_argument("--gpus-per-node", type=optional_int, default=None, help="GPUs per node for multi-node runs") + p.add_argument("--dp-attention", default="false", help="Dashboard flag for DP attention") + p.add_argument("--disagg", default="false", help="Dashboard flag for disaggregated serving") + p.add_argument("--spec-decoding", default="false", help="Dashboard flag for speculative decoding") p.add_argument("--precision", required=True, help="Precision tag (e.g. fp8, bf16)") p.add_argument("--model", required=True, help="HF model identifier (fallback if raw json lacks model_id)") p.add_argument("--image", required=True, help="Docker image used for the run") diff --git a/lib/parse_workload.py b/lib/parse_workload.py index b6b108e..96ae19b 100644 --- a/lib/parse_workload.py +++ b/lib/parse_workload.py @@ -10,6 +10,10 @@ Image precedence: VLLM_IMAGE > VLLM_COMMIT > workload `vllm.image` > `vllm/vllm-openai:latest`. When BENCH_ONLY is truthy, lm_eval task names are not validated against the registry (because they will not run). + +Multi-node workloads can set top-level `num_nodes` and `gpus_per_node` +fields, plus `vllm_bench.metadata` values that control dashboard labels +and the throughput denominator used for per-GPU metrics. """ import os @@ -141,6 +145,28 @@ def find(*names): return tp * dp +def parse_int_arg(serve_args: str, *names: str) -> int | None: + toks = serve_args.split() + for i, t in enumerate(toks): + if "=" in t: + key, _, val = t.partition("=") + if key in names: + try: + return int(val) + except ValueError: + return None + elif t in names and i + 1 < len(toks): + try: + return int(toks[i + 1]) + except ValueError: + return None + return None + + +def bool_text(value: object) -> str: + return "true" if str(value).lower() in {"1", "true", "yes"} else "false" + + def precision_from_model(model: str) -> str: name = model.lower() for marker in ("fp4", "fp8", "int4", "int8", "bf16", "fp16"): @@ -149,10 +175,10 @@ def precision_from_model(model: str) -> str: return "bf16" -def validate_tasks(tasks: list, path: str) -> None: +def validate_tasks(tasks: list, path: str, bench_only: bool = False) -> None: if not tasks: sys.exit(f"{path}: missing or empty `lm_eval.tasks`") - skip_registry = env_truthy("BENCH_ONLY") + skip_registry = bench_only or env_truthy("BENCH_ONLY") known = set() if skip_registry else known_task_names() for t in tasks: extra = set(t) - TASK_FIELDS @@ -303,6 +329,7 @@ def main(path: str) -> None: tasks = lm_eval.get("tasks") or [] bfcl = data.get("bfcl") or {} bench_configs = bench.get("configs") or [] + bench_only = bool_text(data.get("bench_only", False)) if not tasks and not bench_configs and not bfcl: sys.exit( @@ -310,7 +337,7 @@ def main(path: str) -> None: ) if tasks: - validate_tasks(tasks, path) + validate_tasks(tasks, path, bench_only == "true") serve_args = vllm.get("serve_args") or "" if bfcl: @@ -326,22 +353,59 @@ def main(path: str) -> None: if tp is None: tp = parse_tp(serve_args) + num_nodes = data.get("num_nodes") or metadata.get("num_nodes") or 1 + gpus_per_node = data.get("gpus_per_node") or metadata.get("gpus_per_node") + num_gpus = data.get("num_gpus") or metadata.get("num_gpus") + if gpus_per_node is None and num_gpus and num_nodes: + try: + if int(num_gpus) % int(num_nodes) == 0: + gpus_per_node = int(num_gpus) // int(num_nodes) + except (TypeError, ValueError, ZeroDivisionError): + pass + if num_gpus is None and gpus_per_node: + try: + num_gpus = int(num_nodes) * int(gpus_per_node) + except (TypeError, ValueError): + pass + + is_multinode = metadata.get("is_multinode") + if is_multinode is None: + try: + is_multinode = int(num_nodes) > 1 + except (TypeError, ValueError): + is_multinode = False + + gpu_count = ( + metadata.get("gpu_count") + or metadata.get("per_gpu_denominator") + or num_gpus + or tp + ) + emit("NAME", data.get("name", "")) emit("IMAGE", image) emit("VLLM_COMMIT", vllm_commit) emit("MODEL", vllm.get("model", "")) emit("SERVE_ARGS", serve_args) emit("SERVER_RUNTIME", profile.get("server_runtime", "docker")) + emit("NUM_NODES", num_nodes) + emit("GPUS_PER_NODE", gpus_per_node or "") + emit("NUM_GPUS", num_gpus or "") emit("ENV", "\n".join(f"{k}={fmt(v)}" for k, v in env.items())) emit("LM_EVAL_TASKS_TSV", task_tsv(tasks, lm_eval.get("model_args") or {})) emit("VLLM_BENCH_TSV", bench_tsv(bench_configs, path)) emit("BFCL_TSV", bfcl_tsv(bfcl) if bfcl else "") + emit("BENCH_ONLY", bench_only) emit("BENCH_DEVICE", metadata.get("device") or gpu.lower()) emit("BENCH_TP", tp) - emit( - "BENCH_PRECISION", - metadata.get("precision") or precision_from_model(vllm.get("model") or ""), - ) + emit("BENCH_DP", metadata.get("dp") or parse_int_arg(serve_args, "--data-parallel-size", "-dp", "--dp") or "") + emit("BENCH_EP", metadata.get("ep") or ("1" if "--enable-expert-parallel" in serve_args else "")) + emit("BENCH_GPU_COUNT", gpu_count) + emit("BENCH_IS_MULTINODE", bool_text(is_multinode)) + emit("BENCH_DP_ATTENTION", bool_text(metadata.get("dp_attention", False))) + emit("BENCH_DISAGG", bool_text(metadata.get("disagg", False))) + emit("BENCH_SPEC_DECODING", bool_text(metadata.get("spec_decoding", False))) + emit("BENCH_PRECISION", metadata.get("precision") or precision_from_model(vllm.get("model") or "")) if __name__ == "__main__": diff --git a/lib/run.sh b/lib/run.sh index 78f88e5..f753ddf 100755 --- a/lib/run.sh +++ b/lib/run.sh @@ -18,11 +18,16 @@ source "$DIR/run_vllm_bench.sh" WORKLOAD_EXPORTS="$(python3 "$DIR/parse_workload.py" "$WORKLOAD")" eval "$WORKLOAD_EXPORTS" export WORKLOAD_IMAGE WORKLOAD_VLLM_COMMIT WORKLOAD_SERVER_RUNTIME +export WORKLOAD_NAME WORKLOAD_NUM_NODES WORKLOAD_GPUS_PER_NODE WORKLOAD_NUM_GPUS + +while IFS= read -r kv; do + [[ -z "$kv" ]] && continue + export "$kv" +done <<< "$WORKLOAD_ENV" PORT=8000 CONTAINER="perf-eval-${WORKLOAD_NAME}-$$" RESULTS_DIR="results/${WORKLOAD_NAME}" -BASE_URL="http://localhost:${PORT}" BENCH_TRUST_REMOTE_CODE=false if [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code([[:space:]]|$) ]] || [[ "$WORKLOAD_SERVE_ARGS" =~ (^|[[:space:]])--trust-remote-code=(true|True|1|yes|Yes)([[:space:]]|$) ]]; then @@ -34,7 +39,8 @@ trap 'stop_server "$CONTAINER"' EXIT start_server "$CONTAINER" "$PORT" "$WORKLOAD_IMAGE" "$WORKLOAD_MODEL" \ "$WORKLOAD_SERVE_ARGS" "$WORKLOAD_ENV" "$WORKLOAD_SERVER_RUNTIME" -wait_healthy "$PORT" +BASE_URL="${VLLM_BASE_URL:-http://127.0.0.1:${PORT}}" +wait_healthy "$BASE_URL" # vllm bench serve runs first so we can validate perf flow without waiting # on a full lm_eval pass. Each config's raw json lands in @@ -42,7 +48,7 @@ wait_healthy "$PORT" # perf dashboard ingest endpoint. while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset speed_category; do [[ -z "$bname" ]] && continue - run_vllm_bench "$CONTAINER" "$PORT" "$WORKLOAD_MODEL" \ + run_vllm_bench "$CONTAINER" "$BASE_URL" "$WORKLOAD_MODEL" \ "$bname" "$backend" "$dataset" "$isl" "$osl" "$nprompts" \ "$conc" "$speed_subset" "$speed_category" \ "$BENCH_TRUST_REMOTE_CODE" "$RESULTS_DIR" @@ -51,13 +57,22 @@ while IFS=$'\t' read -r bname backend dataset isl osl nprompts conc speed_subset --raw-result "${RESULTS_DIR}/bench-${bname}.json" \ --device "$WORKLOAD_BENCH_DEVICE" \ --tp "$WORKLOAD_BENCH_TP" \ + --dp "${WORKLOAD_BENCH_DP:-}" \ + --ep "${WORKLOAD_BENCH_EP:-}" \ + --gpu-count "$WORKLOAD_BENCH_GPU_COUNT" \ + --is-multinode "$WORKLOAD_BENCH_IS_MULTINODE" \ + --num-nodes "${WORKLOAD_NUM_NODES:-}" \ + --gpus-per-node "${WORKLOAD_GPUS_PER_NODE:-}" \ + --dp-attention "$WORKLOAD_BENCH_DP_ATTENTION" \ + --disagg "$WORKLOAD_BENCH_DISAGG" \ + --spec-decoding "$WORKLOAD_BENCH_SPEC_DECODING" \ --precision "$WORKLOAD_BENCH_PRECISION" \ --model "$WORKLOAD_MODEL" \ --image "$WORKLOAD_IMAGE" \ --isl "$isl" --osl "$osl" --conc "$conc" || true done <<< "$WORKLOAD_VLLM_BENCH_TSV" -if [[ "${BENCH_ONLY:-}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then +if [[ "${BENCH_ONLY:-${WORKLOAD_BENCH_ONLY:-}}" =~ ^([Tt][Rr][Uu][Ee]|1|[Yy][Ee][Ss])$ ]]; then echo "--- :stopwatch: BENCH_ONLY set; skipping lm_eval and bfcl tasks" exit 0 fi diff --git a/lib/run_vllm_bench.sh b/lib/run_vllm_bench.sh index edda073..fa18f5a 100644 --- a/lib/run_vllm_bench.sh +++ b/lib/run_vllm_bench.sh @@ -2,7 +2,7 @@ # Source this from run.sh. # # Usage: -# run_vllm_bench \ +# run_vllm_bench \ # \ # \ # @@ -24,6 +24,47 @@ pip_install_quiet() { fi } +resolve_slurm_container_runtime() { + local runtime=${PERF_EVAL_BENCH_CLIENT_CONTAINER_RUNTIME:-${PERF_EVAL_SLURM_CONTAINER_RUNTIME:-auto}} + if [[ "$runtime" != "auto" ]]; then + printf '%s\n' "$runtime" + return + fi + if srun --help 2>&1 | grep -q -- "--container-image"; then + printf 'pyxis\n' + else + printf 'none\n' + fi +} + +resolve_slurm_container_workdir() { + if [[ -n "${PERF_EVAL_SLURM_CONTAINER_WORKDIR:-}" ]]; then + printf '%s\n' "$PERF_EVAL_SLURM_CONTAINER_WORKDIR" + return + fi + [[ -n "${PERF_EVAL_SLURM_CONTAINER_MOUNTS:-}" ]] || return + + local pwd_real + pwd_real="$(pwd -P)" + local mount host_path container_path rest + IFS=',' read -ra mounts <<< "$PERF_EVAL_SLURM_CONTAINER_MOUNTS" + for mount in "${mounts[@]}"; do + host_path="${mount%%:*}" + rest="${mount#*:}" + [[ "$rest" != "$mount" ]] || continue + container_path="${rest%%:*}" + [[ -n "$host_path" && -n "$container_path" ]] || continue + if [[ "$pwd_real" == "$host_path" ]]; then + printf '%s\n' "$container_path" + return + fi + if [[ "$pwd_real" == "$host_path"/* ]]; then + printf '%s%s\n' "$container_path" "${pwd_real#"$host_path"}" + return + fi + done +} + prepare_speed_bench_dataset() { local container=$1 runtime=$2 subset=$3 category=$4 data_dir=$5 @@ -59,7 +100,7 @@ PY # Docker runtime: ship the data into the container and make sure pandas is # available there (vLLM's SpeedBench loads the JSONL via pandas). - if [[ "$runtime" != "native" ]]; then + if [[ "$runtime" != "native" && "$runtime" != slurm* ]]; then docker exec "$container" mkdir -p "$data_dir" docker cp "${data_dir}/." "${container}:${data_dir}/" if ! docker exec "$container" python3 -c 'import pandas' 2>/dev/null; then @@ -71,8 +112,55 @@ PY fi } +run_bench_command() { + local runtime=$1 + shift + if [[ "$runtime" == slurm* && "${PERF_EVAL_BENCH_CLIENT_RUNTIME:-local}" == "slurm" ]]; then + local container_runtime + container_runtime="$(resolve_slurm_container_runtime)" + local container_workdir + container_workdir="$(resolve_slurm_container_workdir)" + local srun_args=(--ntasks=1) + case "$container_runtime" in + pyxis|container) + srun_args+=(--container-image "$WORKLOAD_IMAGE") + if [[ -n "${PERF_EVAL_SLURM_CONTAINER_MOUNTS:-}" ]]; then + srun_args+=(--container-mounts "$PERF_EVAL_SLURM_CONTAINER_MOUNTS") + fi + if [[ -n "$container_workdir" ]]; then + srun_args+=(--container-workdir "$container_workdir") + fi + if [[ "${PERF_EVAL_SLURM_NO_CONTAINER_REMAP_ROOT:-1}" == "1" ]]; then + srun_args+=(--no-container-remap-root) + fi + ;; + none|native) + ;; + *) + echo "unsupported PERF_EVAL_BENCH_CLIENT_CONTAINER_RUNTIME=$container_runtime" >&2 + return 2 + ;; + esac + if [[ -n "${PERF_EVAL_SLURM_MPI:-pmix}" ]]; then + srun_args+=(--mpi "${PERF_EVAL_SLURM_MPI:-pmix}") + fi + if [[ -n "${PERF_EVAL_BENCH_CLIENT_EXTRA_SRUN_ARGS:-}" ]]; then + # shellcheck disable=SC2206 # Buildkite env intentionally supplies words. + extra_srun_args=($PERF_EVAL_BENCH_CLIENT_EXTRA_SRUN_ARGS) + srun_args+=("${extra_srun_args[@]}") + fi + local cmd="" + for arg in "$@"; do + cmd+=" $(printf "%q" "$arg")" + done + srun "${srun_args[@]}" bash -lc "$cmd" + return + fi + "$@" +} + run_vllm_bench() { - local container=$1 port=$2 model=$3 name=$4 backend=$5 dataset=$6 + local container=$1 base_url=$2 model=$3 name=$4 backend=$5 dataset=$6 local input_len=$7 output_len=$8 num_prompts=$9 max_concurrency=${10} local speed_bench_dataset_subset=${11} speed_bench_category=${12} local trust_remote_code=${13} outdir=${14} @@ -88,13 +176,18 @@ run_vllm_bench() { mkdir -p "$outdir" local cmd=(vllm bench serve) - [[ "$runtime" != "native" ]] && cmd=(docker exec "$container" "${cmd[@]}") + if [[ "$runtime" != "native" && "$runtime" != slurm* ]]; then + cmd=(docker exec "$container" "${cmd[@]}") + fi if [[ -n "$backend" ]]; then - cmd+=(--backend "$backend" --base-url "http://127.0.0.1:${port}") + cmd+=(--backend "$backend" --base-url "$base_url") [[ "$backend" == "openai-chat" ]] && cmd+=(--endpoint /v1/chat/completions) else - cmd+=(--host 127.0.0.1 --port "$port") + local host_port="${base_url#http://}" + host_port="${host_port#https://}" + host_port="${host_port%%/*}" + cmd+=(--host "${host_port%:*}" --port "${host_port##*:}") fi cmd+=( @@ -134,15 +227,17 @@ run_vllm_bench() { ;; esac - if [[ "$runtime" == "native" ]]; then + if [[ "$runtime" == "native" || "$runtime" == slurm* ]]; then cmd+=(--save-result --result-filename "$host_json") else cmd+=(--save-result --result-filename "$in_container_json") fi - "${cmd[@]}" + run_bench_command "$runtime" "${cmd[@]}" - [[ "$runtime" != "native" ]] && docker cp "${container}:${in_container_json}" "$host_json" + if [[ "$runtime" != "native" && "$runtime" != slurm* ]]; then + docker cp "${container}:${in_container_json}" "$host_json" + fi python3 - "$host_json" "$num_prompts" <<'PY' import json, sys diff --git a/lib/server.sh b/lib/server.sh index 2d8ce49..8953817 100644 --- a/lib/server.sh +++ b/lib/server.sh @@ -2,7 +2,7 @@ # # Functions: # start_server [runtime] -# wait_healthy [timeout_s=1500] +# wait_healthy [timeout_s=1500] # stop_server # # `env` is a newline-separated list of KEY=VALUE pairs. For Docker runtime, @@ -13,7 +13,40 @@ # # After start_server, vLLM logs are streamed to stdout (prefixed with `[vllm]`) # so build output reflects server startup progress in real time. The streamer's -# PID is held in $VLLM_LOGS_PID; stop_server kills it. +# PID is held in $VLLM_LOGS_PID; stop_server kills its process group when +# supported so pipeline children such as `tail -F` do not survive teardown. +# +# For `runtime=slurm`, start_server delegates to lib/slurm_launch_vllm.sh by +# default. Custom launchers can be provided with PERF_EVAL_SLURM_LAUNCHER. The +# launcher must submit a long-running server job, write an endpoint env file, +# and print the Slurm job id on stdout. + +start_log_stream() { + local source_cmd=$1 + if command -v setsid >/dev/null 2>&1; then + setsid bash -c "$source_cmd | stdbuf -oL -eL sed 's/^/[vllm] /'" & + VLLM_LOGS_PID=$! + VLLM_LOGS_PGID=$VLLM_LOGS_PID + else + bash -c "$source_cmd | stdbuf -oL -eL sed 's/^/[vllm] /'" & + VLLM_LOGS_PID=$! + VLLM_LOGS_PGID="" + fi +} + +stop_log_stream() { + if [[ -z "${VLLM_LOGS_PID:-}" ]]; then + return + fi + if [[ -n "${VLLM_LOGS_PGID:-}" ]]; then + kill -- "-$VLLM_LOGS_PGID" 2>/dev/null || true + else + kill "$VLLM_LOGS_PID" 2>/dev/null || true + fi + wait "$VLLM_LOGS_PID" 2>/dev/null || true + VLLM_LOGS_PID="" + VLLM_LOGS_PGID="" +} start_server() { local container=$1 port=$2 image=$3 model=$4 serve_args=$5 env=$6 runtime=${7:-docker} @@ -29,9 +62,65 @@ start_server() { # shellcheck disable=SC2086 # serve_args intentionally word-split vllm serve "$model" --port "$port" $serve_args >"$log_file" 2>&1 & VLLM_SERVER_PID=$! + VLLM_BASE_URL="http://127.0.0.1:${port}" echo "--- :memo: streaming vllm logs" - ( tail -f "$log_file" 2>/dev/null | stdbuf -oL -eL sed 's/^/[vllm] /' ) & - VLLM_LOGS_PID=$! + start_log_stream "tail -f $(printf "%q" "$log_file") 2>/dev/null" + return + fi + + if [[ "$runtime" == "slurm" || "$runtime" == "slurm_pyxis" || "$runtime" == "srt_slurm" ]]; then + local results_dir="results/${WORKLOAD_NAME:-$container}" + local launcher="${PERF_EVAL_SLURM_LAUNCHER:-$DIR/slurm_launch_vllm.sh}" + local endpoint_file="${PERF_EVAL_SLURM_ENDPOINT_FILE:-${results_dir}/${container}.endpoint}" + local env_file="${results_dir}/${container}.env" + mkdir -p "$results_dir" + printf '%s\n' "$env" > "$env_file" + while IFS= read -r kv; do + [[ -z "$kv" ]] && continue + case "$kv" in + PERF_EVAL_SLURM_*=*) + export "$kv" + ;; + esac + done <<< "$env" + + [[ -x "$launcher" ]] || { echo "Slurm launcher not executable: $launcher" >&2; return 2; } + + echo "--- :satellite: submitting vLLM Slurm server" + VLLM_SLURM_JOB_ID="$( + PERF_EVAL_CONTAINER_NAME="$container" \ + PERF_EVAL_PORT="$port" \ + PERF_EVAL_IMAGE="$image" \ + PERF_EVAL_MODEL="$model" \ + PERF_EVAL_SERVE_ARGS="$serve_args" \ + PERF_EVAL_ENV_FILE="$env_file" \ + PERF_EVAL_ENDPOINT_FILE="$endpoint_file" \ + PERF_EVAL_RESULTS_DIR="$results_dir" \ + PERF_EVAL_NUM_NODES="${WORKLOAD_NUM_NODES:-1}" \ + PERF_EVAL_GPUS_PER_NODE="${WORKLOAD_GPUS_PER_NODE:-${WORKLOAD_NUM_GPUS:-1}}" \ + PERF_EVAL_NUM_GPUS="${WORKLOAD_NUM_GPUS:-}" \ + "$launcher" + )" + export VLLM_SLURM_JOB_ID + + local timeout="${PERF_EVAL_SLURM_ENDPOINT_TIMEOUT_S:-600}" + local deadline=$(( $(date +%s) + timeout )) + while [[ ! -s "$endpoint_file" ]]; do + if (( $(date +%s) >= deadline )); then + echo "Slurm server did not publish endpoint file: $endpoint_file" >&2 + return 1 + fi + sleep 5 + done + # shellcheck disable=SC1090 + source "$endpoint_file" + VLLM_BASE_URL="${VLLM_BASE_URL:-http://${VLLM_SERVER_HOST}:${VLLM_SERVER_PORT:-$port}}" + export VLLM_BASE_URL + echo "server endpoint: $VLLM_BASE_URL" + if [[ -n "${VLLM_SLURM_LOG_FILE:-}" ]]; then + echo "--- :memo: streaming vllm Slurm logs" + start_log_stream "tail -F $(printf "%q" "$VLLM_SLURM_LOG_FILE") 2>/dev/null" + fi return fi @@ -54,24 +143,30 @@ start_server() { docker run -d --rm --name "$container" "${docker_args[@]}" \ "$image" \ "$model" --port "$port" $serve_args + VLLM_BASE_URL="http://127.0.0.1:${port}" # Install pytest to avoid cupy.testing import failure during torch.compile docker exec "$container" pip install -q pytest 2>/dev/null || true echo "--- :memo: streaming vllm logs" - ( docker logs -f "$container" 2>&1 | stdbuf -oL -eL sed 's/^/[vllm] /' ) & - VLLM_LOGS_PID=$! + start_log_stream "docker logs -f $(printf "%q" "$container") 2>&1" } wait_healthy() { - local port=$1 timeout=${2:-3600} - echo "+++ :hourglass: waiting for /health (timeout ${timeout}s)" + local target=$1 timeout=${2:-3600} + local health_url + if [[ "$target" == http://* || "$target" == https://* ]]; then + health_url="${target%/}/health" + else + health_url="http://localhost:${target}/health" + fi + echo "+++ :hourglass: waiting for ${health_url} (timeout ${timeout}s)" local now start deadline next_status elapsed start=$(date +%s) deadline=$(( start + timeout )) next_status=$(( start + 60 )) while (( $(date +%s) < deadline )); do - if curl -fs "http://localhost:${port}/health" >/dev/null 2>&1; then + if curl -fs "$health_url" >/dev/null 2>&1; then echo "server healthy" return 0 fi @@ -94,13 +189,13 @@ wait_healthy() { stop_server() { local container=$1 - if [[ -n "${VLLM_LOGS_PID:-}" ]]; then - kill "$VLLM_LOGS_PID" 2>/dev/null || true - wait "$VLLM_LOGS_PID" 2>/dev/null || true - fi + stop_log_stream if [[ -n "${VLLM_SERVER_PID:-}" ]]; then kill "$VLLM_SERVER_PID" 2>/dev/null || true wait "$VLLM_SERVER_PID" 2>/dev/null || true fi + if [[ -n "${VLLM_SLURM_JOB_ID:-}" ]]; then + scancel "$VLLM_SLURM_JOB_ID" >/dev/null 2>&1 || true + fi docker rm -f "$container" >/dev/null 2>&1 || true } diff --git a/lib/slurm_launch_vllm.sh b/lib/slurm_launch_vllm.sh new file mode 100755 index 0000000..84bbfc3 --- /dev/null +++ b/lib/slurm_launch_vllm.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +# Submit a long-running vLLM server job to Slurm and print the Slurm job id. +# +# Inputs are supplied by lib/server.sh through PERF_EVAL_* environment vars. +# The default path launches non-Ray `vllm serve` through Slurm. If Pyxis is +# available it can run in a Slurm container; clusters with their own launch +# wrapper can use PERF_EVAL_SLURM_SERVER_COMMAND. +set -euo pipefail + +require() { + local name=$1 + local value=${!name:-} + [[ -n "$value" ]] || { echo "missing required env $name" >&2; exit 2; } +} + +shell_quote() { + printf "%q" "$1" +} + +is_truthy() { + case "${1,,}" in + 1|true|yes|on) return 0 ;; + *) return 1 ;; + esac +} + +resolve_container_runtime() { + local runtime=${PERF_EVAL_SLURM_CONTAINER_RUNTIME:-auto} + if [[ "$runtime" != "auto" ]]; then + printf '%s\n' "$runtime" + return + fi + if srun --help 2>&1 | grep -q -- "--container-image"; then + printf 'pyxis\n' + else + printf 'none\n' + fi +} + +resolve_container_workdir() { + if [[ -n "${PERF_EVAL_SLURM_CONTAINER_WORKDIR:-}" ]]; then + printf '%s\n' "$PERF_EVAL_SLURM_CONTAINER_WORKDIR" + return + fi + [[ -n "${PERF_EVAL_SLURM_CONTAINER_MOUNTS:-}" ]] || return + + local pwd_real + pwd_real="$(pwd -P)" + local mount host_path container_path rest + IFS=',' read -ra mounts <<< "$PERF_EVAL_SLURM_CONTAINER_MOUNTS" + for mount in "${mounts[@]}"; do + host_path="${mount%%:*}" + rest="${mount#*:}" + [[ "$rest" != "$mount" ]] || continue + container_path="${rest%%:*}" + [[ -n "$host_path" && -n "$container_path" ]] || continue + if [[ "$pwd_real" == "$host_path" ]]; then + printf '%s\n' "$container_path" + return + fi + if [[ "$pwd_real" == "$host_path"/* ]]; then + printf '%s%s\n' "$container_path" "${pwd_real#"$host_path"}" + return + fi + done +} + +is_integer() { + [[ "$1" =~ ^[0-9]+$ ]] +} + +require PERF_EVAL_CONTAINER_NAME +require PERF_EVAL_PORT +require PERF_EVAL_MODEL +require PERF_EVAL_ENDPOINT_FILE +require PERF_EVAL_RESULTS_DIR + +mkdir -p "$PERF_EVAL_RESULTS_DIR" + +script="${PERF_EVAL_RESULTS_DIR}/${PERF_EVAL_CONTAINER_NAME}.sbatch.sh" +log_file="${PERF_EVAL_RESULTS_DIR}/${PERF_EVAL_CONTAINER_NAME}-%j.log" +nodes="${PERF_EVAL_NUM_NODES:-1}" +gpus_per_node="${PERF_EVAL_GPUS_PER_NODE:-${PERF_EVAL_NUM_GPUS:-1}}" +slurm_gpus_per_node="${PERF_EVAL_SLURM_GPUS_PER_NODE:-$gpus_per_node}" +ntasks_per_node="${PERF_EVAL_SLURM_NTASKS_PER_NODE:-1}" +job_name="${PERF_EVAL_SLURM_JOB_NAME:-$PERF_EVAL_CONTAINER_NAME}" +container_runtime="$(resolve_container_runtime)" +request_gpus="${PERF_EVAL_SLURM_REQUEST_GPUS:-1}" +slurm_gres="${PERF_EVAL_SLURM_GRES:-}" +container_workdir="$(resolve_container_workdir)" +vllm_distributed_backend="${PERF_EVAL_SLURM_VLLM_DISTRIBUTED_BACKEND:-}" +if [[ -z "$vllm_distributed_backend" ]] && is_integer "$nodes" && (( nodes > 1 )); then + vllm_distributed_backend=mp +fi + +serve_cmd="vllm serve $(shell_quote "$PERF_EVAL_MODEL") --host 0.0.0.0 --port $(shell_quote "$PERF_EVAL_PORT")" +if [[ -n "${PERF_EVAL_SERVE_ARGS:-}" ]]; then + serve_cmd+=" ${PERF_EVAL_SERVE_ARGS}" +fi + +server_command="${PERF_EVAL_SLURM_SERVER_COMMAND:-}" +server_command_block="" +if [[ -z "$server_command" ]]; then + srun_common_args=() + if is_truthy "$request_gpus"; then + if [[ -n "$slurm_gres" ]]; then + srun_common_args+=(--gres "$slurm_gres") + elif [[ -n "$slurm_gpus_per_node" ]]; then + srun_common_args+=(--gpus-per-node "$slurm_gpus_per_node") + fi + fi + case "$container_runtime" in + pyxis|container) + require PERF_EVAL_IMAGE + srun_common_args+=(--container-image "$PERF_EVAL_IMAGE") + if [[ -n "${PERF_EVAL_SLURM_CONTAINER_MOUNTS:-}" ]]; then + srun_common_args+=(--container-mounts "$PERF_EVAL_SLURM_CONTAINER_MOUNTS") + fi + if [[ -n "$container_workdir" ]]; then + srun_common_args+=(--container-workdir "$container_workdir") + fi + if [[ "${PERF_EVAL_SLURM_NO_CONTAINER_REMAP_ROOT:-1}" == "1" ]]; then + srun_common_args+=(--no-container-remap-root) + fi + ;; + none|native) + ;; + *) + echo "unsupported PERF_EVAL_SLURM_CONTAINER_RUNTIME=$container_runtime" >&2 + exit 2 + ;; + esac + if [[ -n "${PERF_EVAL_SLURM_MPI:-pmix}" ]]; then + srun_common_args+=(--mpi "${PERF_EVAL_SLURM_MPI:-pmix}") + fi + if [[ -n "${PERF_EVAL_SLURM_EXTRA_SRUN_ARGS:-}" ]]; then + # shellcheck disable=SC2206 # Buildkite env intentionally supplies words. + extra_srun_args=($PERF_EVAL_SLURM_EXTRA_SRUN_ARGS) + srun_common_args+=("${extra_srun_args[@]}") + fi + + srun_common="" + for arg in "${srun_common_args[@]}"; do + srun_common+=" $(shell_quote "$arg")" + done + + if [[ "$vllm_distributed_backend" == "mp" ]] && is_integer "$nodes" && (( nodes > 1 )); then + server_command_block=$(cat </dev/null 2>&1 || true + done +} +trap cleanup_servers EXIT INT TERM + +if ((\${#perf_eval_slurm_hosts[@]} < $(shell_quote "$nodes"))); then + echo "expected $(shell_quote "$nodes") Slurm hosts, got \${#perf_eval_slurm_hosts[@]}" >&2 + exit 2 +fi +master_addr="\${perf_eval_slurm_hosts[0]}" +base_serve_cmd=$(shell_quote "$serve_cmd") +for rank in \$(seq 0 $((nodes - 1))); do + node="\${perf_eval_slurm_hosts[\$rank]}" + rank_cmd="\$base_serve_cmd --distributed-executor-backend mp --nnodes $(shell_quote "$nodes") --master-addr \$master_addr --node-rank \$rank" + if ((rank > 0)); then + rank_cmd+=" --headless" + fi + srun --nodes=1 --ntasks=1 --nodelist "\$node"$srun_common bash -lc "\$rank_cmd" & + server_pids+=("\$!") +done + +wait -n "\${server_pids[@]}" +status=\$? +cleanup_servers +wait "\${server_pids[@]}" >/dev/null 2>&1 || true +exit "\$status" +BLOCK +) + else + server_command="srun" + server_command+=" --ntasks=1" + server_command+="$srun_common" + server_command+=" bash -lc $(shell_quote "$serve_cmd")" + fi +fi + +cat > "$script" <