diff --git a/.buildkite/generate_pipeline.py b/.buildkite/generate_pipeline.py index 82d3703..ce6d8bf 100644 --- a/.buildkite/generate_pipeline.py +++ b/.buildkite/generate_pipeline.py @@ -114,7 +114,7 @@ def b200_k8s_plugin(image, num_gpus): "env": [ {"name": "VLLM_USAGE_SOURCE", "value": "ci-test"}, {"name": "NCCL_CUMEM_HOST_ENABLE", "value": "0"}, - {"name": "HF_HOME", "value": "/mnt/shared/hf_cache"}, + {"name": "HF_HOME", "value": "/raid/hf_cache"}, { "name": "HF_TOKEN", "valueFrom": { diff --git a/README.md b/README.md index 93bfb1b..a085f0f 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,9 @@ vllm: # how the server is brought up env: # optional; merged over the GPU profile's env SOME_VAR: value serve_args: >- # appended to `vllm serve `; word-split - -dp 8 --enable-expert-parallel - --trust-remote-code + -dp 8 --enable-expert-parallel # NOTE: word-split is unquoted, so do NOT wrap + --trust-remote-code # values in quotes and keep each token space-free. + --speculative-config {"method":"eagle3","num_speculative_tokens":3} # JSON args: no quotes, no spaces lm_eval: # accuracy tasks (optional) model_args: # workload-level defaults, merged into every task diff --git a/lib/gpu_profiles.yaml b/lib/gpu_profiles.yaml index 329080c..ce285c2 100644 --- a/lib/gpu_profiles.yaml +++ b/lib/gpu_profiles.yaml @@ -8,7 +8,10 @@ H200: B200: queue: b200-k8s - hf_home: /mnt/shared/hf_cache + # Cache models on the 28T /raid volume, not the ~1.8T root disk. /mnt/shared + # is NOT a shared mount on the DGX B200 nodes — it's a plain dir on root, so + # caching there fills root and triggers kubelet disk-pressure pod eviction. + hf_home: /raid/hf_cache server_runtime: native env: VLLM_DEEP_GEMM_WARMUP: skip diff --git a/lib/run_vllm_bench.sh b/lib/run_vllm_bench.sh index 3c7fe7d..ef72114 100644 --- a/lib/run_vllm_bench.sh +++ b/lib/run_vllm_bench.sh @@ -57,9 +57,10 @@ PY fi test -s "${data_dir}/${subset}.jsonl" - # Docker runtime: ship the data into the container and make sure pandas is - # available there (vLLM's SpeedBench loads the JSONL via pandas). + # vLLM's SpeedBench loads the JSONL via pandas, which the release image + # lacks. Ensure pandas is importable by whatever Python runs `vllm bench`. if [[ "$runtime" != "native" ]]; then + # Docker runtime: ship the data into the container and install there. docker exec "$container" mkdir -p "$data_dir" docker cp "${data_dir}/." "${container}:${data_dir}/" if ! docker exec "$container" python3 -c 'import pandas' 2>/dev/null; then @@ -68,6 +69,19 @@ PY 'PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --quiet pandas \ || PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --user --quiet pandas' fi + else + # Native runtime: `vllm bench serve` runs under the interpreter backing the + # `vllm` CLI (the container's system Python), not the job .venv, so pandas + # installed into the .venv above isn't visible to it. + local vllm_bin vllm_py + vllm_bin="$(command -v vllm || true)" + vllm_py="$(awk 'NR==1{sub(/^#![[:space:]]*/,""); print $1; exit}' "$vllm_bin" 2>/dev/null)" + [[ -x "$vllm_py" ]] || vllm_py="$(command -v python3)" + if ! "$vllm_py" -c 'import pandas' 2>/dev/null; then + echo "--- :python: installing pandas for native vllm bench ($vllm_py)" + PIP_BREAK_SYSTEM_PACKAGES=1 "$vllm_py" -m pip install --quiet pandas \ + || PIP_BREAK_SYSTEM_PACKAGES=1 "$vllm_py" -m pip install --user --quiet pandas + fi fi } diff --git a/workloads/deepseek_v3_2_b200.yaml b/workloads/deepseek_v3_2_b200.yaml new file mode 100644 index 0000000..574699a --- /dev/null +++ b/workloads/deepseek_v3_2_b200.yaml @@ -0,0 +1,45 @@ +# DeepSeek-V3.2 NVFP4 on B200 (TP=8; sparse attention, auto MLA backend) +# Recipe: https://recipes.vllm.ai/nvidia/DeepSeek-V3.2-NVFP4 +name: deepseek_v3_2-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: nvidia/DeepSeek-V3.2-NVFP4 + serve_args: >- + --trust-remote-code + --kernel-config.enable_flashinfer_autotune=False + --kv-cache-dtype fp8 + --tensor-parallel-size 8 + --tokenizer-mode deepseek_v32 + --tool-call-parser deepseek_v32 + --enable-auto-tool-choice + --reasoning-parser deepseek_v3 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 4096 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/deepseek_v4_flash_b200.yaml b/workloads/deepseek_v4_flash_b200.yaml index 694ab37..34ac08b 100644 --- a/workloads/deepseek_v4_flash_b200.yaml +++ b/workloads/deepseek_v4_flash_b200.yaml @@ -1,4 +1,4 @@ -# DeepSeek-V4-Flash on B200 (TP=2 × DP=4 + EP, deep_gemm_mega_moe, MTP spec-decode) +# DeepSeek-V4-Flash on B200 (TP=2 × DP=4 + EP, deep_gemm, MTP spec-decode) name: deepseek_v4_flash-b200 gpu: B200 num_gpus: 8 @@ -13,7 +13,7 @@ vllm: --max-model-len 32768 --kv-cache-dtype fp8 --block-size 256 - --moe-backend deep_gemm_mega_moe + --moe-backend deep_gemm --attention_config.use_fp4_indexer_cache=True --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 diff --git a/workloads/deepseek_v4_pro_5_b200.yaml b/workloads/deepseek_v4_pro_5_b200.yaml new file mode 100644 index 0000000..92b4cf8 --- /dev/null +++ b/workloads/deepseek_v4_pro_5_b200.yaml @@ -0,0 +1,49 @@ +# DeepSeek-V4-Pro on B200 (DP=8 + EP, deep_gemm) +name: deepseek_v4_pro-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: deepseek-ai/DeepSeek-V4-Pro + serve_args: >- + --data-parallel-size 8 + --enable-expert-parallel + --max-model-len 32768 + --max-num-seqs 512 + --max-num-batched-tokens 512 + --kv-cache-dtype fp8 + --block-size 256 + --moe-backend deep_gemm + --attention_config.use_fp4_indexer_cache=True + --tokenizer-mode deepseek_v4 + --gpu-memory-utilization 0.95 + --tool-call-parser deepseek_v4 + --reasoning-parser deepseek_v4 + --enable-auto-tool-choice + --trust-remote-code + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/glm_5_1_b200.yaml b/workloads/glm_5_1_b200.yaml new file mode 100644 index 0000000..4ddc6e3 --- /dev/null +++ b/workloads/glm_5_1_b200.yaml @@ -0,0 +1,44 @@ +# GLM-5.1 NVFP4 on B200 (TP=8) +# Recipe: https://recipes.vllm.ai/nvidia/GLM-5.1-NVFP4 +name: glm_5_1-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: nvidia/GLM-5.1-NVFP4 + serve_args: >- + --trust-remote-code + --chat-template-content-format=string + --kv-cache-dtype fp8 + --tensor-parallel-size 8 + --tool-call-parser glm47 + --enable-auto-tool-choice + --reasoning-parser glm45 + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 40960 + max_gen_toks: 32768 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/kimi_k2_5_b200.yaml b/workloads/kimi_k2_5_b200.yaml new file mode 100644 index 0000000..f43efb7 --- /dev/null +++ b/workloads/kimi_k2_5_b200.yaml @@ -0,0 +1,46 @@ +# Kimi-K2.5 NVFP4 on B200 (TP=8, Eagle3 speculative) +# Recipe: https://recipes.vllm.ai/nvidia/Kimi-K2.5-NVFP4 +name: kimi_k2_5-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: nvidia/Kimi-K2.5-NVFP4 + serve_args: >- + --trust-remote-code + --kv-cache-dtype fp8 + --tensor-parallel-size 8 + --attention-config.mla_prefill_backend=TRTLLM_RAGGED + --tool-call-parser kimi_k2 + --enable-auto-tool-choice + --reasoning-parser kimi_k2 + --speculative-config {"model":"lightseekorg/kimi-k2.5-eagle3-mla","method":"eagle3","num_speculative_tokens":3} + --mm-encoder-tp-mode data + env: + VLLM_USE_FLASHINFER_MOE_FP4: "1" + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 40960 + max_gen_toks: 32768 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/minimax_m2_5_b200.yaml b/workloads/minimax_m2_5_b200.yaml new file mode 100644 index 0000000..053c65d --- /dev/null +++ b/workloads/minimax_m2_5_b200.yaml @@ -0,0 +1,41 @@ +# MiniMax-M2.5 on B200 (TP=4) +# Recipe: https://recipes.vllm.ai/MiniMaxAI/MiniMax-M2.5 +name: minimax_m2_5-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: MiniMaxAI/MiniMax-M2.5 + serve_args: >- + --trust-remote-code + --tensor-parallel-size 4 + --tool-call-parser minimax_m2 + --enable-auto-tool-choice + --reasoning-parser minimax_m2 + --compilation-config {"mode":3,"cudagraph_mode":"PIECEWISE","pass_config":{"fuse_minimax_qk_norm":true}} + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 40960 + max_gen_toks: 32768 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/nemotron_3_super_5_b200.yaml b/workloads/nemotron_3_super_5_b200.yaml new file mode 100644 index 0000000..2b72450 --- /dev/null +++ b/workloads/nemotron_3_super_5_b200.yaml @@ -0,0 +1,40 @@ +# Nemotron-3-Super-120B-A12B NVFP4 on B200 (TP=1) +# Recipe: https://recipes.vllm.ai/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 +name: nemotron_3_super-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 + serve_args: >- + --trust-remote-code + --tensor-parallel-size 1 + --enable-auto-tool-choice + --tool-call-parser qwen3_coder + --reasoning-parser nemotron_v3 + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy diff --git a/workloads/qwen3_5_b200.yaml b/workloads/qwen3_5_b200.yaml new file mode 100644 index 0000000..3db44bc --- /dev/null +++ b/workloads/qwen3_5_b200.yaml @@ -0,0 +1,46 @@ +# Qwen3.5-397B-A17B NVFP4 on B200 (TP=8) +# Recipe: https://recipes.vllm.ai/nvidia/Qwen3.5-397B-A17B-NVFP4 +name: qwen3_5-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + serve_args: >- + --trust-remote-code + --kv-cache-dtype fp8 + --tensor-parallel-size 8 + --enable-auto-tool-choice + --tool-call-parser qwen3_coder + --reasoning-parser qwen3 + --mm-encoder-tp-mode data + env: + VLLM_DEEP_GEMM_WARMUP: skip + VLLM_USE_DEEP_GEMM: "0" + VLLM_FLASHINFER_MOE_BACKEND: latency + VLLM_USE_FLASHINFER_MOE_FP4: "1" + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 40960 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai-chat + dataset: speed_bench + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 + speed_bench_dataset_subset: throughput_8k + speed_bench_category: low_entropy