Add B200 workload variants for all H200-only models

.buildkite/generate_pipeline.py

@@ -114,7 +114,7 @@ def b200_k8s_plugin(image, num_gpus):
                         "env": [
                             {"name": "VLLM_USAGE_SOURCE", "value": "ci-test"},
                             {"name": "NCCL_CUMEM_HOST_ENABLE", "value": "0"},
-                            {"name": "HF_HOME", "value": "/mnt/shared/hf_cache"},
+                            {"name": "HF_HOME", "value": "/raid/hf_cache"},
                             {
                                 "name": "HF_TOKEN",
                                 "valueFrom": {

README.md

@@ -45,8 +45,9 @@ vllm:                    # how the server is brought up
   env:                                  # optional; merged over the GPU profile's env
     SOME_VAR: value
   serve_args: >-                        # appended to `vllm serve <model>`; word-split
-    -dp 8 --enable-expert-parallel
-    --trust-remote-code
+    -dp 8 --enable-expert-parallel       # NOTE: word-split is unquoted, so do NOT wrap
+    --trust-remote-code                  # values in quotes and keep each token space-free.
+    --speculative-config {"method":"eagle3","num_speculative_tokens":3}  # JSON args: no quotes, no spaces
 
 lm_eval:                 # accuracy tasks (optional)
   model_args:            # workload-level defaults, merged into every task

lib/gpu_profiles.yaml

@@ -8,7 +8,10 @@ H200:
 
 B200:
   queue: b200-k8s
-  hf_home: /mnt/shared/hf_cache
+  # Cache models on the 28T /raid volume, not the ~1.8T root disk. /mnt/shared
+  # is NOT a shared mount on the DGX B200 nodes — it's a plain dir on root, so
+  # caching there fills root and triggers kubelet disk-pressure pod eviction.
+  hf_home: /raid/hf_cache
   server_runtime: native
   env:
     VLLM_DEEP_GEMM_WARMUP: skip

lib/run_vllm_bench.sh

@@ -57,9 +57,10 @@ PY
   fi
   test -s "${data_dir}/${subset}.jsonl"
 
-  # Docker runtime: ship the data into the container and make sure pandas is
-  # available there (vLLM's SpeedBench loads the JSONL via pandas).
+  # vLLM's SpeedBench loads the JSONL via pandas, which the release image
+  # lacks. Ensure pandas is importable by whatever Python runs `vllm bench`.
   if [[ "$runtime" != "native" ]]; then
+    # Docker runtime: ship the data into the container and install there.
     docker exec "$container" mkdir -p "$data_dir"
     docker cp "${data_dir}/." "${container}:${data_dir}/"
     if ! docker exec "$container" python3 -c 'import pandas' 2>/dev/null; then
@@ -68,6 +69,19 @@ PY
         'PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --quiet pandas \
           || PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --user --quiet pandas'
     fi
+  else
+    # Native runtime: `vllm bench serve` runs under the interpreter backing the
+    # `vllm` CLI (the container's system Python), not the job .venv, so pandas
+    # installed into the .venv above isn't visible to it.
+    local vllm_bin vllm_py
+    vllm_bin="$(command -v vllm || true)"
+    vllm_py="$(awk 'NR==1{sub(/^#![[:space:]]*/,""); print $1; exit}' "$vllm_bin" 2>/dev/null)"
+    [[ -x "$vllm_py" ]] || vllm_py="$(command -v python3)"
+    if ! "$vllm_py" -c 'import pandas' 2>/dev/null; then
+      echo "--- :python: installing pandas for native vllm bench ($vllm_py)"
+      PIP_BREAK_SYSTEM_PACKAGES=1 "$vllm_py" -m pip install --quiet pandas \
+        || PIP_BREAK_SYSTEM_PACKAGES=1 "$vllm_py" -m pip install --user --quiet pandas
+    fi
   fi
 }
 

workloads/deepseek_v3_2_b200.yaml

@@ -0,0 +1,45 @@
+# DeepSeek-V3.2 NVFP4 on B200 (TP=8; sparse attention, auto MLA backend)
+# Recipe: https://recipes.vllm.ai/nvidia/DeepSeek-V3.2-NVFP4
+name: deepseek_v3_2-b200
+gpu: B200
+num_gpus: 8
+nightly: true
+
+vllm:
+  model: nvidia/DeepSeek-V3.2-NVFP4
+  serve_args: >-
+    --trust-remote-code
+    --kernel-config.enable_flashinfer_autotune=False
+    --kv-cache-dtype fp8
+    --tensor-parallel-size 8
+    --tokenizer-mode deepseek_v32
+    --tool-call-parser deepseek_v32
+    --enable-auto-tool-choice
+    --reasoning-parser deepseek_v3
+  env:
+    VLLM_USE_FLASHINFER_MOE_FP4: "1"
+
+lm_eval:
+  model_args:
+    tokenized_requests: false
+    tokenizer_backend: null
+    timeout: 6000
+  tasks:
+    - name: gsm8k
+      num_fewshot: 5
+      model_args:
+        num_concurrent: 64
+        max_length: 32768
+        max_gen_toks: 4096
+
+vllm_bench:
+  configs:
+    - name: 8k-in-1k-out-conc-128
+      backend: openai-chat
+      dataset: speed_bench
+      input_len: 8192
+      output_len: 1024
+      num_prompts: 512
+      max_concurrency: 128
+      speed_bench_dataset_subset: throughput_8k
+      speed_bench_category: low_entropy

workloads/deepseek_v4_flash_b200.yaml

@@ -1,4 +1,4 @@
-# DeepSeek-V4-Flash on B200 (TP=2 × DP=4 + EP, deep_gemm_mega_moe, MTP spec-decode)
+# DeepSeek-V4-Flash on B200 (TP=2 × DP=4 + EP, deep_gemm, MTP spec-decode)
 name: deepseek_v4_flash-b200
 gpu: B200
 num_gpus: 8
@@ -13,7 +13,7 @@ vllm:
     --max-model-len 32768
     --kv-cache-dtype fp8
     --block-size 256
-    --moe-backend deep_gemm_mega_moe
+    --moe-backend deep_gemm
     --attention_config.use_fp4_indexer_cache=True
     --tokenizer-mode deepseek_v4
     --tool-call-parser deepseek_v4

workloads/deepseek_v4_pro_5_b200.yaml

@@ -0,0 +1,49 @@
+# DeepSeek-V4-Pro on B200 (DP=8 + EP, deep_gemm)
+name: deepseek_v4_pro-b200
+gpu: B200
+num_gpus: 8
+nightly: true
+
+vllm:
+  model: deepseek-ai/DeepSeek-V4-Pro
+  serve_args: >-
+    --data-parallel-size 8
+    --enable-expert-parallel
+    --max-model-len 32768
+    --max-num-seqs 512
+    --max-num-batched-tokens 512
+    --kv-cache-dtype fp8
+    --block-size 256
+    --moe-backend deep_gemm
+    --attention_config.use_fp4_indexer_cache=True
+    --tokenizer-mode deepseek_v4
+    --gpu-memory-utilization 0.95
+    --tool-call-parser deepseek_v4
+    --reasoning-parser deepseek_v4
+    --enable-auto-tool-choice
+    --trust-remote-code
+
+lm_eval:
+  model_args:
+    tokenized_requests: false
+    tokenizer_backend: null
+    timeout: 6000
+  tasks:
+    - name: gsm8k
+      num_fewshot: 5
+      model_args:
+        num_concurrent: 64
+        max_length: 32768
+        max_gen_toks: 8192
+
+vllm_bench:
+  configs:
+    - name: 8k-in-1k-out-conc-128
+      backend: openai-chat
+      dataset: speed_bench
+      input_len: 8192
+      output_len: 1024
+      num_prompts: 512
+      max_concurrency: 128
+      speed_bench_dataset_subset: throughput_8k
+      speed_bench_category: low_entropy

workloads/glm_5_1_b200.yaml

@@ -0,0 +1,44 @@
+# GLM-5.1 NVFP4 on B200 (TP=8)
+# Recipe: https://recipes.vllm.ai/nvidia/GLM-5.1-NVFP4
+name: glm_5_1-b200
+gpu: B200
+num_gpus: 8
+nightly: true
+
+vllm:
+  model: nvidia/GLM-5.1-NVFP4
+  serve_args: >-
+    --trust-remote-code
+    --chat-template-content-format=string
+    --kv-cache-dtype fp8
+    --tensor-parallel-size 8
+    --tool-call-parser glm47
+    --enable-auto-tool-choice
+    --reasoning-parser glm45
+  env:
+    VLLM_USE_FLASHINFER_MOE_FP4: "1"
+
+lm_eval:
+  model_args:
+    tokenized_requests: false
+    tokenizer_backend: null
+    timeout: 6000
+  tasks:
+    - name: gsm8k
+      num_fewshot: 5
+      model_args:
+        num_concurrent: 64
+        max_length: 40960
+        max_gen_toks: 32768
+
+vllm_bench:
+  configs:
+    - name: 8k-in-1k-out-conc-128
+      backend: openai-chat
+      dataset: speed_bench
+      input_len: 8192
+      output_len: 1024
+      num_prompts: 512
+      max_concurrency: 128
+      speed_bench_dataset_subset: throughput_8k
+      speed_bench_category: low_entropy

workloads/kimi_k2_5_b200.yaml

@@ -0,0 +1,46 @@
+# Kimi-K2.5 NVFP4 on B200 (TP=8, Eagle3 speculative)
+# Recipe: https://recipes.vllm.ai/nvidia/Kimi-K2.5-NVFP4
+name: kimi_k2_5-b200
+gpu: B200
+num_gpus: 8
+nightly: true
+
+vllm:
+  model: nvidia/Kimi-K2.5-NVFP4
+  serve_args: >-
+    --trust-remote-code
+    --kv-cache-dtype fp8
+    --tensor-parallel-size 8
+    --attention-config.mla_prefill_backend=TRTLLM_RAGGED
+    --tool-call-parser kimi_k2
+    --enable-auto-tool-choice
+    --reasoning-parser kimi_k2
+    --speculative-config {"model":"lightseekorg/kimi-k2.5-eagle3-mla","method":"eagle3","num_speculative_tokens":3}
+    --mm-encoder-tp-mode data
+  env:
+    VLLM_USE_FLASHINFER_MOE_FP4: "1"
+
+lm_eval:
+  model_args:
+    tokenized_requests: false
+    tokenizer_backend: null
+    timeout: 6000
+  tasks:
+    - name: gsm8k
+      num_fewshot: 5
+      model_args:
+        num_concurrent: 64
+        max_length: 40960
+        max_gen_toks: 32768
+
+vllm_bench:
+  configs:
+    - name: 8k-in-1k-out-conc-128
+      backend: openai-chat
+      dataset: speed_bench
+      input_len: 8192
+      output_len: 1024
+      num_prompts: 512
+      max_concurrency: 128
+      speed_bench_dataset_subset: throughput_8k
+      speed_bench_category: low_entropy

workloads/minimax_m2_5_b200.yaml

@@ -0,0 +1,41 @@
+# MiniMax-M2.5 on B200 (TP=4)
+# Recipe: https://recipes.vllm.ai/MiniMaxAI/MiniMax-M2.5
+name: minimax_m2_5-b200
+gpu: B200
+num_gpus: 8
+nightly: true
+
+vllm:
+  model: MiniMaxAI/MiniMax-M2.5
+  serve_args: >-
+    --trust-remote-code
+    --tensor-parallel-size 4
+    --tool-call-parser minimax_m2
+    --enable-auto-tool-choice
+    --reasoning-parser minimax_m2
+    --compilation-config {"mode":3,"cudagraph_mode":"PIECEWISE","pass_config":{"fuse_minimax_qk_norm":true}}
+
+lm_eval:
+  model_args:
+    tokenized_requests: false
+    tokenizer_backend: null
+    timeout: 6000
+  tasks:
+    - name: gsm8k
+      num_fewshot: 5
+      model_args:
+        num_concurrent: 64
+        max_length: 40960
+        max_gen_toks: 32768
+
+vllm_bench:
+  configs:
+    - name: 8k-in-1k-out-conc-128
+      backend: openai-chat
+      dataset: speed_bench
+      input_len: 8192
+      output_len: 1024
+      num_prompts: 512
+      max_concurrency: 128
+      speed_bench_dataset_subset: throughput_8k
+      speed_bench_category: low_entropy

workloads/nemotron_3_super_5_b200.yaml

@@ -0,0 +1,40 @@
+# Nemotron-3-Super-120B-A12B NVFP4 on B200 (TP=1)
+# Recipe: https://recipes.vllm.ai/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4
+name: nemotron_3_super-b200
+gpu: B200
+num_gpus: 8
+nightly: true
+
+vllm:
+  model: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4
+  serve_args: >-
+    --trust-remote-code
+    --tensor-parallel-size 1
+    --enable-auto-tool-choice
+    --tool-call-parser qwen3_coder
+    --reasoning-parser nemotron_v3
+
+lm_eval:
+  model_args:
+    tokenized_requests: false
+    tokenizer_backend: null
+    timeout: 6000
+  tasks:
+    - name: gsm8k
+      num_fewshot: 5
+      model_args:
+        num_concurrent: 64
+        max_length: 32768
+        max_gen_toks: 8192
+
+vllm_bench:
+  configs:
+    - name: 8k-in-1k-out-conc-128
+      backend: openai-chat
+      dataset: speed_bench
+      input_len: 8192
+      output_len: 1024
+      num_prompts: 512
+      max_concurrency: 128
+      speed_bench_dataset_subset: throughput_8k
+      speed_bench_category: low_entropy

workloads/qwen3_5_b200.yaml

@@ -0,0 +1,46 @@
+# Qwen3.5-397B-A17B NVFP4 on B200 (TP=8)
+# Recipe: https://recipes.vllm.ai/nvidia/Qwen3.5-397B-A17B-NVFP4
+name: qwen3_5-b200
+gpu: B200
+num_gpus: 8
+nightly: true
+
+vllm:
+  model: nvidia/Qwen3.5-397B-A17B-NVFP4
+  serve_args: >-
+    --trust-remote-code
+    --kv-cache-dtype fp8
+    --tensor-parallel-size 8
+    --enable-auto-tool-choice
+    --tool-call-parser qwen3_coder
+    --reasoning-parser qwen3
+    --mm-encoder-tp-mode data
+  env:
+    VLLM_DEEP_GEMM_WARMUP: skip
+    VLLM_USE_DEEP_GEMM: "0"
+    VLLM_FLASHINFER_MOE_BACKEND: latency
+    VLLM_USE_FLASHINFER_MOE_FP4: "1"
+
+lm_eval:
+  model_args:
+    tokenized_requests: false
+    tokenizer_backend: null
+    timeout: 6000
+  tasks:
+    - name: gsm8k
+      num_fewshot: 5
+      model_args:
+        num_concurrent: 64
+        max_length: 40960
+
+vllm_bench:
+  configs:
+    - name: 8k-in-1k-out-conc-128
+      backend: openai-chat
+      dataset: speed_bench
+      input_len: 8192
+      output_len: 1024
+      num_prompts: 512
+      max_concurrency: 128
+      speed_bench_dataset_subset: throughput_8k
+      speed_bench_category: low_entropy