From 8d175948df5d97e9cffb026ab9a9b6d5fa0f8cfa Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 15:54:47 +0800
Subject: [PATCH 01/12] feat(dsv4): scaffold FP8 MI300X SGLang bring-up
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

New SKU — no prior DeepSeek-V4-Pro data on gfx942. Extrapolated per
recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro from the same-model MI355X
entry (search-space shape, deepseek_v4 flags), the same-SKU
dsr1-fp8-mi300x-sglang (gfx942 AITER infra), and the H200 dsv4 FP8 path
(--quantization deepseek_v4_fp8). gfx942 has no native FP4, so the ~960GB
mixed checkpoint runs in FP8 (~1.05TB), which only fits TP8 on 8x192GB —
MI355X TP4 bands dropped. Sweep: TP-only low-latency (conc 1-32) + DP-attn/
EP8 throughput (conc 64-2048), 1k1k + 8k1k.

Unvalidated on-cluster: needs debug-runs to confirm the mi30x image has the
DeepseekV4 class + --attention-backend dsv4 / --quantization deepseek_v4_fp8
on gfx942.

中文：搭建 MI300X 上 DeepSeek-V4-Pro FP8 单节点 SGLang 脚手架（依据
recipes.vllm.ai 及同型号 MI355X、同 SKU dsr1-mi300x 外推；FP8 仅 TP8 可
放下故去掉 TP4）。尚未集群验证。
---
 .../fixed_seq_len/dsv4_fp8_mi300x.sh          | 141 ++++++++++++++++++
 configs/amd-master.yaml                       |  34 +++++
 perf-changelog.yaml                           |   9 ++
 3 files changed, 184 insertions(+)
 create mode 100755 benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
new file mode 100755
index 000000000..73abf8112
--- /dev/null
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+# DeepSeek-V4-Pro FP8 single-node on MI300X (gfx942) via SGLang.
+#
+# EXTRAPOLATED bring-up recipe (unvalidated on-cluster as of 2026-07),
+# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus:
+#   * same model, adjacent SKU: dsv4_fp4_mi355x_sglang.sh (search-space
+#     shape, DP-attention + EP path, deepseek_v4 model flags, SWA/page-size)
+#   * same SKU, different model: dsr1_fp8_mi300x.sh (gfx942 infra: AITER,
+#     MEC-firmware scratch-reclaim guard)
+#   * same model, FP8 path: the H200 dsv4 recipe (--quantization
+#     deepseek_v4_fp8) — gfx942 has no native FP4, so the FP4-MoE checkpoint
+#     is run in FP8.
+#
+# Sizing: the ~960GB mixed checkpoint is ~1.05TB in FP8, so only TP8 fits
+# 8x192GB (TP4 = 768GB does not). The config restricts to TP8 accordingly.
+#
+# Debug-runs must confirm before this leaves bring-up:
+#   1. The mi30x image carries the DeepseekV4 model class.
+#   2. --attention-backend dsv4 exists on gfx942 (else fall back to aiter).
+#   3. --quantization deepseek_v4_fp8 loads on ROCm (else use the AITER FP8
+#      MoE path without an explicit quant flag).
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars \
+    MODEL \
+    TP \
+    DP_ATTENTION \
+    EP_SIZE \
+    CONC \
+    ISL \
+    OSL \
+    RANDOM_RANGE_RATIO \
+    RESULT_FILENAME \
+    MAX_MODEL_LEN
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+  echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi
+
+# If the machine runs a MEC FW older than 177, RCCL cannot reclaim some
+# memory; disable that feature to avoid crashes (see dsr1_fp8_mi300x.sh).
+version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'`
+if [[ "$version" == "" || $version -lt 177 ]]; then
+  export HSA_NO_SCRATCH_RECLAIM=1
+fi
+
+# gfx942 AITER infra (dsr1_fp8_mi300x.sh) + deepseek_v4 model env (mi355x).
+export SGLANG_USE_AITER=1
+export SGLANG_AITER_MLA_PERSIST=1
+export SGLANG_DEFAULT_THINKING=1
+export SGLANG_DSV4_REASONING_EFFORT=max
+export AITER_BF16_FP8_MOE_BOUND=0
+
+SERVER_LOG=/workspace/server.log
+
+EVAL_CONTEXT_ARGS=""
+if [ "${EVAL_ONLY}" = "true" ]; then
+    setup_eval_context
+    EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN"
+fi
+
+# Start GPU monitoring (power, temperature, clocks every second)
+start_gpu_monitor
+
+# DP_ATTENTION=true runs DP-attention with expert parallel (dp-size = TP),
+# mirroring dsv4_fp4_mi355x_sglang.sh; false runs pure tensor parallel for
+# the low-latency band.
+PARALLEL_ARGS=(--tensor-parallel-size "$TP")
+CHUNKED_PREFILL_SIZE=$ISL
+if [ "${DP_ATTENTION}" = "true" ]; then
+    export SGLANG_SHARED_EXPERT_TP1=1
+    export SGLANG_DP_SHARED_EXPERT_LOCAL=1
+    export SGLANG_DP_USE_GATHERV=1
+    export SGLANG_DP_USE_REDUCE_SCATTER=1
+    export GPU_MAX_HW_QUEUES=5
+
+    CHUNKED_PREFILL_SIZE=$((ISL * TP))
+    PARALLEL_ARGS+=(
+        --dp "$TP"
+        --enable-dp-attention
+        --enable-prefill-delayer
+        --enable-two-batch-overlap
+    )
+fi
+if [ "${EP_SIZE:-1}" -gt 1 ]; then
+    PARALLEL_ARGS+=(--ep-size "$EP_SIZE")
+fi
+
+set -x
+sglang serve \
+    --model-path $MODEL --served-model-name $MODEL \
+    --host=0.0.0.0 --port $PORT \
+    "${PARALLEL_ARGS[@]}" \
+    --trust-remote-code \
+    --quantization deepseek_v4_fp8 \
+    --kv-cache-dtype fp8_e4m3 \
+    --attention-backend dsv4 \
+    --disable-radix-cache \
+    --disable-shared-experts-fusion \
+    --page-size 256 \
+    --mem-fraction-static 0.90 \
+    --swa-full-tokens-ratio 0.15 \
+    --cuda-graph-max-bs ${CONC} \
+    --max-running-requests ${CONC} \
+    --context-length $MAX_MODEL_LEN \
+    --chunked-prefill-size $CHUNKED_PREFILL_SIZE \
+    --tool-call-parser deepseekv4 \
+    --reasoning-parser deepseek-v4 \
+    --chat-template "$(dirname "$0")/../chat_templates/deepseek_v4_thinking.jinja" \
+    --watchdog-timeout 1800 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 &
+
+SERVER_PID=$!
+
+# Wait for server to be ready
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+run_benchmark_serving \
+    --model "$MODEL" \
+    --port "$PORT" \
+    --backend vllm \
+    --input-len "$ISL" \
+    --output-len "$OSL" \
+    --random-range-ratio "$RANDOM_RANGE_RATIO" \
+    --num-prompts "$((CONC * 10))" \
+    --max-concurrency "$CONC" \
+    --result-filename "$RESULT_FILENAME" \
+    --result-dir /workspace/
+
+# After throughput, run evaluation only if RUN_EVAL is true
+if [ "${RUN_EVAL}" = "true" ]; then
+    run_eval --framework lm-eval --port "$PORT"
+    append_lm_eval_summary
+fi
+
+# Stop GPU monitoring
+stop_gpu_monitor
+set +x
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index 4b1ea4aed..e812b0ada 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -119,6 +119,40 @@ dsr1-fp8-mi325x-sglang:
       search-space:
       - { tp: 8, conc-start: 4, conc-end: 64 }
 
+# DeepSeek-V4-Pro FP8 single-node on MI300X (gfx942) via SGLang.
+# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07).
+# Search-space shape mirrors the same-model MI355X entry (dsv4-fp4-mi355x-sglang):
+# a TP-only low-latency band + a DP-attention + EP throughput band. gfx942 has
+# no native FP4, so this runs FP8 (the ~960GB mixed checkpoint is ~1.05TB in
+# FP8) — which only fits TP8 on 8x192GB, so the MI355X TP4 bands are dropped.
+# Image follows the same-model MI355X's sglang-rocm dated-tag family (mi30x
+# variant); launch script dsv4_fp8_mi300x.sh carries the deepseek_v4 flags +
+# gfx942 AITER infra. debug-runs must confirm the image carries the DeepseekV4
+# class and that --attention-backend dsv4 / --quantization deepseek_v4_fp8
+# work on gfx942 before this merges.
+dsv4-fp8-mi300x-sglang:
+  image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: mi300x
+  precision: fp8
+  framework: sglang
+  multinode: false
+  scenarios:
+    fixed-seq-len:
+    - isl: 1024
+      osl: 1024
+      search-space:
+      # Low latency: TP-only (dp-attn off).
+      - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32 }
+      # Throughput: DP-attention + EP8 (mirrors the MI355X tp8 dp-attn band).
+      - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 2048 }
+    - isl: 8192
+      osl: 1024
+      search-space:
+      - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32 }
+      - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 2048 }
+
 dsr1-fp8-mi355x-sglang:
   image: lmsysorg/sglang:v0.5.12-rocm700-mi35x
   model: deepseek-ai/DeepSeek-R1-0528
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 57d510dd1..33be791f3 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4750,3 +4750,12 @@
     - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928"
     - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137
+
+- config-keys:
+    - dsv4-fp8-mi300x-sglang
+  description:
+    - "Add DeepSeek-V4-Pro FP8 single-node MI300X SGLang benchmark (new SKU, previously no dsv4 data on gfx942)"
+    - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi300x-sglang for gfx942 AITER infra; FP8 path (--quantization deepseek_v4_fp8) follows the H200 dsv4 recipe since gfx942 has no native FP4"
+    - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8 on 8x192GB"
+    - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2194

From cc8d5feaec89a8a74ee733b1184d10e87d0ac381 Mon Sep 17 00:00:00 2001
From: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 03:50:38 -0700
Subject: [PATCH 02/12] s

---
 perf-changelog.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 634c33ebc..98f1ccbad 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4752,7 +4752,6 @@
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137
 
 - config-keys:
-<<<<<<< dsv4-mi300x-fp8-bringup
     - dsv4-fp8-mi300x-sglang
   description:
     - "Add DeepSeek-V4-Pro FP8 single-node MI300X SGLang benchmark (new SKU, previously no dsv4 data on gfx942)"
@@ -4760,7 +4759,8 @@
     - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8 on 8x192GB"
     - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2194
-=======
+
+- config-keys:
     - dsv4-fp4-mi355x-sglang
   description:
     - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706"
@@ -4774,4 +4774,3 @@
     - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708"
     - "Clean the export envs"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198
->>>>>>> main

From c1bfd67087809bbfedc3671cb03cfdc180ebcda1 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 19:13:45 +0800
Subject: [PATCH 03/12] fix(dsv4): drop invalid --quantization from MI300X
 sglang script
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

sglang argparse rejected --quantization deepseek_v4_fp8 (a vLLM-only
method), crashing the canary before server start (run 29326787628).
Remove the flag: sglang reads the modelopt quant config from the
checkpoint and runs it via the AITER MoE path, matching the H200 and
MI355X dsv4 sglang recipes (which pass no --quantization).

中文：删除 MI300X sglang 脚本中无效的 --quantization deepseek_v4_fp8
（该方法仅 vLLM 支持，sglang argparse 直接报错）。改为不传 --quantization，
由 sglang 从 checkpoint 读取 modelopt 量化配置并经 AITER MoE 执行，与 H200、
MI355X 的 dsv4 sglang 配方一致。
---
 .../single_node/fixed_seq_len/dsv4_fp8_mi300x.sh   | 14 ++++++++------
 perf-changelog.yaml                                |  3 ++-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
index 73abf8112..229485c2a 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
@@ -8,9 +8,12 @@
 #     shape, DP-attention + EP path, deepseek_v4 model flags, SWA/page-size)
 #   * same SKU, different model: dsr1_fp8_mi300x.sh (gfx942 infra: AITER,
 #     MEC-firmware scratch-reclaim guard)
-#   * same model, FP8 path: the H200 dsv4 recipe (--quantization
-#     deepseek_v4_fp8) — gfx942 has no native FP4, so the FP4-MoE checkpoint
-#     is run in FP8.
+#   * same model + framework: dsv4_fp8_h200_sglang.sh — gfx942 has no native
+#     FP4, so the FP4-MoE checkpoint is run in FP8. Like the H200 sglang
+#     recipe (and the MI355X dsv4 sglang recipe), NO --quantization flag is
+#     passed: sglang reads the modelopt quant config from the checkpoint and
+#     the AITER MoE path (SGLANG_USE_AITER=1) executes it. (--quantization
+#     deepseek_v4_fp8 is a vLLM-only method; sglang argparse rejects it.)
 #
 # Sizing: the ~960GB mixed checkpoint is ~1.05TB in FP8, so only TP8 fits
 # 8x192GB (TP4 = 768GB does not). The config restricts to TP8 accordingly.
@@ -18,8 +21,8 @@
 # Debug-runs must confirm before this leaves bring-up:
 #   1. The mi30x image carries the DeepseekV4 model class.
 #   2. --attention-backend dsv4 exists on gfx942 (else fall back to aiter).
-#   3. --quantization deepseek_v4_fp8 loads on ROCm (else use the AITER FP8
-#      MoE path without an explicit quant flag).
+#   3. The modelopt FP4-MoE checkpoint loads + runs in FP8 via AITER on gfx942
+#      (else pass an explicit valid sglang quant, e.g. --quantization fp8).
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -96,7 +99,6 @@ sglang serve \
     --host=0.0.0.0 --port $PORT \
     "${PARALLEL_ARGS[@]}" \
     --trust-remote-code \
-    --quantization deepseek_v4_fp8 \
     --kv-cache-dtype fp8_e4m3 \
     --attention-backend dsv4 \
     --disable-radix-cache \
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 98f1ccbad..b96e08d07 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4755,7 +4755,8 @@
     - dsv4-fp8-mi300x-sglang
   description:
     - "Add DeepSeek-V4-Pro FP8 single-node MI300X SGLang benchmark (new SKU, previously no dsv4 data on gfx942)"
-    - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi300x-sglang for gfx942 AITER infra; FP8 path (--quantization deepseek_v4_fp8) follows the H200 dsv4 recipe since gfx942 has no native FP4"
+    - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi300x-sglang for gfx942 AITER infra; gfx942 has no native FP4 so the FP4-MoE checkpoint runs in FP8"
+    - "No --quantization flag: sglang reads the modelopt quant config from the checkpoint and the AITER MoE path executes it, matching the H200 and MI355X dsv4 sglang recipes (--quantization deepseek_v4_fp8 is vLLM-only and rejected by sglang argparse)"
     - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8 on 8x192GB"
     - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2194

From f289605830a2fc69f99f58c3bcbc1d8ee8f79a82 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 19:31:16 +0800
Subject: [PATCH 04/12] fix(dsv4): route MI300X MLA through unified_kv_triton
 FlashMLA
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The prior fix cleared argparse but the server was then killed at CUDA-graph
capture (run 29328162558): --attention-backend dsv4 compiles its FP8 MLA
kernel via TileLang, whose InjectSoftwarePipeline pass fails on gfx942
("buffer access dependency ... cannot be reordered").

Set SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton (+ SGLANG_USE_ROCM700A=0),
matching the same-model dsv4-fp4-mi355x-sglang recipe, so MLA runs through the
unified-KV triton path and avoids the failing TileLang kernel. Drop
SGLANG_AITER_MLA_PERSIST (MLA no longer via aiter); keep SGLANG_USE_AITER for
the MoE GEMMs, which were already running fine.

中文：MI300X 上 --attention-backend dsv4 的 FP8 MLA 内核经 TileLang 编译在
gfx942 上失败（软件流水线重排检查），导致 CUDA graph 捕获阶段服务被 kill。
参照同型号 mi355x 配方，设置 SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
改走统一 KV triton FlashMLA 路径以绕开该内核。
---
 .../single_node/fixed_seq_len/dsv4_fp8_mi300x.sh  | 15 ++++++++++++---
 perf-changelog.yaml                               |  1 +
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
index 229485c2a..b40d1709b 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
@@ -20,7 +20,8 @@
 #
 # Debug-runs must confirm before this leaves bring-up:
 #   1. The mi30x image carries the DeepseekV4 model class.
-#   2. --attention-backend dsv4 exists on gfx942 (else fall back to aiter).
+#   2. The unified-KV triton FlashMLA path captures CUDA graphs cleanly on
+#      gfx942 (the TileLang FP8 MLA kernel does not — see the env note below).
 #   3. The modelopt FP4-MoE checkpoint loads + runs in FP8 via AITER on gfx942
 #      (else pass an explicit valid sglang quant, e.g. --quantization fp8).
 
@@ -51,9 +52,17 @@ if [[ "$version" == "" || $version -lt 177 ]]; then
   export HSA_NO_SCRATCH_RECLAIM=1
 fi
 
-# gfx942 AITER infra (dsr1_fp8_mi300x.sh) + deepseek_v4 model env (mi355x).
+# gfx942 AITER infra (dsr1_fp8_mi300x.sh, for the MoE GEMMs) + deepseek_v4
+# model env (dsv4_fp4_mi355x_sglang.sh). SGLANG_HACK_FLASHMLA_BACKEND=
+# unified_kv_triton is load-bearing: without it the dsv4 attention backend
+# compiles its FP8 MLA kernel via TileLang, whose InjectSoftwarePipeline pass
+# fails on gfx942 ("buffer access dependency ... cannot be reordered") and
+# kills the server at CUDA-graph capture. The unified-KV triton FlashMLA path
+# avoids that kernel. (SGLANG_AITER_MLA_PERSIST dropped: MLA no longer runs
+# through aiter.)
 export SGLANG_USE_AITER=1
-export SGLANG_AITER_MLA_PERSIST=1
+export SGLANG_USE_ROCM700A=0
+export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
 export SGLANG_DEFAULT_THINKING=1
 export SGLANG_DSV4_REASONING_EFFORT=max
 export AITER_BF16_FP8_MOE_BOUND=0
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index b96e08d07..9c016bec0 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4757,6 +4757,7 @@
     - "Add DeepSeek-V4-Pro FP8 single-node MI300X SGLang benchmark (new SKU, previously no dsv4 data on gfx942)"
     - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi300x-sglang for gfx942 AITER infra; gfx942 has no native FP4 so the FP4-MoE checkpoint runs in FP8"
     - "No --quantization flag: sglang reads the modelopt quant config from the checkpoint and the AITER MoE path executes it, matching the H200 and MI355X dsv4 sglang recipes (--quantization deepseek_v4_fp8 is vLLM-only and rejected by sglang argparse)"
+    - "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton (+ SGLANG_USE_ROCM700A=0), mirroring dsv4-fp4-mi355x-sglang: the dsv4 attention backend's TileLang FP8 MLA kernel fails to compile on gfx942 (InjectSoftwarePipeline reorder check) and kills the server at CUDA-graph capture; the unified-KV triton FlashMLA path avoids it"
     - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8 on 8x192GB"
     - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2194

From e837ae96e1e17d110305b963637d966549fa0cde Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 19:45:39 +0800
Subject: [PATCH 05/12] fix(dsv4): use triton MoE runner for MI300X (CK
 unsupported on gfx942)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Third bring-up layer: unified_kv_triton cleared the MLA kernel, but the
server was then killed at CUDA-graph capture (run 29329188550) in the AITER
Composable-Kernel fused-MoE (ck_moe_stage1) with "Unsupported kernel config
for moe heuristic dispatch" — CK has no tuned config for dsv4's FP8 MoE
shapes on gfx942. Add --moe-runner-backend triton, which JIT-generates
per-shape and needs no pretuned table. MI355X/gfx950 keeps the CK default
(which has gfx950 configs).

中文：第三层 bring-up 问题——AITER Composable-Kernel 的 fused-MoE 在 gfx942
上没有 dsv4 FP8 MoE 形状的调优配置，CUDA graph 捕获阶段报
"Unsupported kernel config for moe heuristic dispatch"。改用
--moe-runner-backend triton（按形状 JIT 生成，无需预调优表）。
---
 .../single_node/fixed_seq_len/dsv4_fp8_mi300x.sh     | 12 ++++++++++--
 perf-changelog.yaml                                  |  1 +
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
index b40d1709b..281f64b99 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
@@ -18,12 +18,19 @@
 # Sizing: the ~960GB mixed checkpoint is ~1.05TB in FP8, so only TP8 fits
 # 8x192GB (TP4 = 768GB does not). The config restricts to TP8 accordingly.
 #
+# MoE runner: triton, not the default AITER Composable-Kernel path. The CK
+# fused-MoE kernel (ck_moe_stage1) has no tuned config for DeepSeek-V4's FP8
+# MoE shapes on gfx942 and aborts CUDA-graph capture with "Unsupported kernel
+# config for moe heuristic dispatch"; triton JIT-generates per-shape so it
+# needs no pretuned table. (The MI355X/gfx950 recipe uses the CK default,
+# which does have gfx950 configs.)
+#
 # Debug-runs must confirm before this leaves bring-up:
 #   1. The mi30x image carries the DeepseekV4 model class.
 #   2. The unified-KV triton FlashMLA path captures CUDA graphs cleanly on
 #      gfx942 (the TileLang FP8 MLA kernel does not — see the env note below).
-#   3. The modelopt FP4-MoE checkpoint loads + runs in FP8 via AITER on gfx942
-#      (else pass an explicit valid sglang quant, e.g. --quantization fp8).
+#   3. The triton MoE runner supports the dsv4 FP8 MoE shapes on gfx942 (the
+#      AITER CK path does not); confirm throughput is acceptable vs CK.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -110,6 +117,7 @@ sglang serve \
     --trust-remote-code \
     --kv-cache-dtype fp8_e4m3 \
     --attention-backend dsv4 \
+    --moe-runner-backend triton \
     --disable-radix-cache \
     --disable-shared-experts-fusion \
     --page-size 256 \
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 9c016bec0..8e40f663e 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4758,6 +4758,7 @@
     - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi300x-sglang for gfx942 AITER infra; gfx942 has no native FP4 so the FP4-MoE checkpoint runs in FP8"
     - "No --quantization flag: sglang reads the modelopt quant config from the checkpoint and the AITER MoE path executes it, matching the H200 and MI355X dsv4 sglang recipes (--quantization deepseek_v4_fp8 is vLLM-only and rejected by sglang argparse)"
     - "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton (+ SGLANG_USE_ROCM700A=0), mirroring dsv4-fp4-mi355x-sglang: the dsv4 attention backend's TileLang FP8 MLA kernel fails to compile on gfx942 (InjectSoftwarePipeline reorder check) and kills the server at CUDA-graph capture; the unified-KV triton FlashMLA path avoids it"
+    - "--moe-runner-backend triton: the default AITER Composable-Kernel fused-MoE (ck_moe_stage1) has no tuned config for dsv4's FP8 MoE shapes on gfx942 and aborts CUDA-graph capture ('Unsupported kernel config for moe heuristic dispatch'); triton JIT-generates per-shape so it needs no pretuned table (MI355X/gfx950 keeps the CK default, which has gfx950 configs)"
     - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8 on 8x192GB"
     - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2194

From c4035024e2b6b46ee26c13fdd1627d60bcb0c1d7 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 20:14:17 +0800
Subject: [PATCH 06/12] fix(dsv4): switch MI300X dsv4 to vLLM FP8 (sglang
 blocked on gfx942)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The sglang path hit successive gfx942 kernel gaps (nvfp4 MoE, TileLang MLA)
that only have gfx950/MI355X builds. Pivot to vLLM, which runs the checkpoint
in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8) — the same
path the no-FP4 H200 dsv4 vLLM recipe uses. Config/script mirror the same-model
dsv4-fp4-mi355x-vllm (AITER MoE, deepseek_v4 tokenizer/parser, mp executor,
FULL_AND_PIECEWISE compile) + gfx942 AITER infra; TP8 conc 4-512. Config key
renamed dsv4-fp8-mi300x-sglang -> dsv4-fp8-mi300x-vllm.

中文：MI300X dsv4 由 sglang 改为 vLLM。sglang 在 gfx942 上接连遇到只有
gfx950/MI355X 才有的内核缺口（nvfp4 MoE、TileLang MLA）。vLLM 通过
--quantization deepseek_v4_fp8 将 FP4 MoE 反量化为 FP8 运行，与无 FP4 的
H200 dsv4 vLLM 配方一致。配方参照同型号 dsv4-fp4-mi355x-vllm。
---
 .../fixed_seq_len/dsv4_fp8_mi300x.sh          | 138 ++++++------------
 configs/amd-master.yaml                       |  33 ++---
 perf-changelog.yaml                           |  15 +-
 3 files changed, 61 insertions(+), 125 deletions(-)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
index 281f64b99..5bedcdf37 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
@@ -1,36 +1,22 @@
 #!/usr/bin/env bash
+set -eo pipefail
 
-# DeepSeek-V4-Pro FP8 single-node on MI300X (gfx942) via SGLang.
+# DeepSeek-V4-Pro FP8 single-node on MI300X (gfx942) via vLLM.
 #
-# EXTRAPOLATED bring-up recipe (unvalidated on-cluster as of 2026-07),
-# derived per https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro plus:
-#   * same model, adjacent SKU: dsv4_fp4_mi355x_sglang.sh (search-space
-#     shape, DP-attention + EP path, deepseek_v4 model flags, SWA/page-size)
-#   * same SKU, different model: dsr1_fp8_mi300x.sh (gfx942 infra: AITER,
-#     MEC-firmware scratch-reclaim guard)
-#   * same model + framework: dsv4_fp8_h200_sglang.sh — gfx942 has no native
-#     FP4, so the FP4-MoE checkpoint is run in FP8. Like the H200 sglang
-#     recipe (and the MI355X dsv4 sglang recipe), NO --quantization flag is
-#     passed: sglang reads the modelopt quant config from the checkpoint and
-#     the AITER MoE path (SGLANG_USE_AITER=1) executes it. (--quantization
-#     deepseek_v4_fp8 is a vLLM-only method; sglang argparse rejects it.)
+# EXTRAPOLATED bring-up recipe. The sglang path was abandoned: on gfx942
+# (no native FP4) the dsv4 sglang backend's nvfp4 MoE / TileLang-MLA kernels
+# have no gfx942 equivalents (they exist only for gfx950/MI355X). vLLM instead
+# runs the checkpoint in FP8 via --quantization deepseek_v4_fp8, which
+# dequantizes the FP4 MoE experts to FP8 — the same path the H200 dsv4 vLLM
+# recipe uses (H200 is also a no-FP4 SKU). Derived from:
+#   * same model + framework + AMD family: dsv4_fp4_mi355x_vllm.sh (ROCm vLLM
+#     dsv4 structure: AITER MoE, deepseek_v4 tokenizer/parser, mp executor,
+#     FULL_AND_PIECEWISE compile)
+#   * same model, FP8 path: dsv4_fp8_h200.sh (--quantization deepseek_v4_fp8)
+#   * same SKU, different model: minimaxm3_fp8_mi300x.sh (gfx942 vLLM/AITER)
 #
-# Sizing: the ~960GB mixed checkpoint is ~1.05TB in FP8, so only TP8 fits
-# 8x192GB (TP4 = 768GB does not). The config restricts to TP8 accordingly.
-#
-# MoE runner: triton, not the default AITER Composable-Kernel path. The CK
-# fused-MoE kernel (ck_moe_stage1) has no tuned config for DeepSeek-V4's FP8
-# MoE shapes on gfx942 and aborts CUDA-graph capture with "Unsupported kernel
-# config for moe heuristic dispatch"; triton JIT-generates per-shape so it
-# needs no pretuned table. (The MI355X/gfx950 recipe uses the CK default,
-# which does have gfx950 configs.)
-#
-# Debug-runs must confirm before this leaves bring-up:
-#   1. The mi30x image carries the DeepseekV4 model class.
-#   2. The unified-KV triton FlashMLA path captures CUDA graphs cleanly on
-#      gfx942 (the TileLang FP8 MLA kernel does not — see the env note below).
-#   3. The triton MoE runner supports the dsv4 FP8 MoE shapes on gfx942 (the
-#      AITER CK path does not); confirm throughput is acceptable vs CK.
+# The FP4->FP8 dequant roughly doubles the MoE footprint (~1.05 TB total),
+# which fits 8x192 GB only at TP8, so the sweep is TP8-only.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -38,13 +24,12 @@ check_env_vars \
     MODEL \
     TP \
     DP_ATTENTION \
-    EP_SIZE \
     CONC \
     ISL \
     OSL \
+    MAX_MODEL_LEN \
     RANDOM_RANGE_RATIO \
-    RESULT_FILENAME \
-    MAX_MODEL_LEN
+    RESULT_FILENAME
 
 if [[ -n "$SLURM_JOB_ID" ]]; then
   echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
@@ -52,89 +37,51 @@ fi
 
 if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi
 
-# If the machine runs a MEC FW older than 177, RCCL cannot reclaim some
-# memory; disable that feature to avoid crashes (see dsr1_fp8_mi300x.sh).
-version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'`
-if [[ "$version" == "" || $version -lt 177 ]]; then
-  export HSA_NO_SCRATCH_RECLAIM=1
+if [ -n "$ROCR_VISIBLE_DEVICES" ]; then
+    export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
 fi
 
-# gfx942 AITER infra (dsr1_fp8_mi300x.sh, for the MoE GEMMs) + deepseek_v4
-# model env (dsv4_fp4_mi355x_sglang.sh). SGLANG_HACK_FLASHMLA_BACKEND=
-# unified_kv_triton is load-bearing: without it the dsv4 attention backend
-# compiles its FP8 MLA kernel via TileLang, whose InjectSoftwarePipeline pass
-# fails on gfx942 ("buffer access dependency ... cannot be reordered") and
-# kills the server at CUDA-graph capture. The unified-KV triton FlashMLA path
-# avoids that kernel. (SGLANG_AITER_MLA_PERSIST dropped: MLA no longer runs
-# through aiter.)
-export SGLANG_USE_AITER=1
-export SGLANG_USE_ROCM700A=0
-export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
-export SGLANG_DEFAULT_THINKING=1
-export SGLANG_DSV4_REASONING_EFFORT=max
-export AITER_BF16_FP8_MOE_BOUND=0
+export VLLM_ROCM_USE_AITER=1
+export VLLM_ROCM_USE_AITER_MOE=1
 
 SERVER_LOG=/workspace/server.log
 
-EVAL_CONTEXT_ARGS=""
 if [ "${EVAL_ONLY}" = "true" ]; then
     setup_eval_context
-    EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN"
+    MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN"
 fi
 
-# Start GPU monitoring (power, temperature, clocks every second)
 start_gpu_monitor
 
-# DP_ATTENTION=true runs DP-attention with expert parallel (dp-size = TP),
-# mirroring dsv4_fp4_mi355x_sglang.sh; false runs pure tensor parallel for
-# the low-latency band.
-PARALLEL_ARGS=(--tensor-parallel-size "$TP")
-CHUNKED_PREFILL_SIZE=$ISL
+PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
 if [ "${DP_ATTENTION}" = "true" ]; then
-    export SGLANG_SHARED_EXPERT_TP1=1
-    export SGLANG_DP_SHARED_EXPERT_LOCAL=1
-    export SGLANG_DP_USE_GATHERV=1
-    export SGLANG_DP_USE_REDUCE_SCATTER=1
-    export GPU_MAX_HW_QUEUES=5
-
-    CHUNKED_PREFILL_SIZE=$((ISL * TP))
-    PARALLEL_ARGS+=(
-        --dp "$TP"
-        --enable-dp-attention
-        --enable-prefill-delayer
-        --enable-two-batch-overlap
-    )
+    PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
 fi
+
+EP_ARGS=()
 if [ "${EP_SIZE:-1}" -gt 1 ]; then
-    PARALLEL_ARGS+=(--ep-size "$EP_SIZE")
+    EP_ARGS=(--enable-expert-parallel)
 fi
 
 set -x
-sglang serve \
-    --model-path $MODEL --served-model-name $MODEL \
-    --host=0.0.0.0 --port $PORT \
+vllm serve $MODEL --port $PORT \
     "${PARALLEL_ARGS[@]}" \
+    "${EP_ARGS[@]}" \
+    --quantization deepseek_v4_fp8 \
+    --async-scheduling \
+    --no-enable-prefix-caching \
+    --distributed-executor-backend mp \
+    --gpu-memory-utilization 0.9 \
+    --max-model-len "$MAX_MODEL_LEN" \
+    --kv-cache-dtype fp8 \
     --trust-remote-code \
-    --kv-cache-dtype fp8_e4m3 \
-    --attention-backend dsv4 \
-    --moe-runner-backend triton \
-    --disable-radix-cache \
-    --disable-shared-experts-fusion \
-    --page-size 256 \
-    --mem-fraction-static 0.90 \
-    --swa-full-tokens-ratio 0.15 \
-    --cuda-graph-max-bs ${CONC} \
-    --max-running-requests ${CONC} \
-    --context-length $MAX_MODEL_LEN \
-    --chunked-prefill-size $CHUNKED_PREFILL_SIZE \
-    --tool-call-parser deepseekv4 \
-    --reasoning-parser deepseek-v4 \
-    --chat-template "$(dirname "$0")/../chat_templates/deepseek_v4_thinking.jinja" \
-    --watchdog-timeout 1800 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 &
+    --moe-backend aiter \
+    --tokenizer-mode deepseek_v4 \
+    --reasoning-parser deepseek_v4 \
+    --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 &
 
 SERVER_PID=$!
 
-# Wait for server to be ready
 wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
 
 run_benchmark_serving \
@@ -147,14 +94,13 @@ run_benchmark_serving \
     --num-prompts "$((CONC * 10))" \
     --max-concurrency "$CONC" \
     --result-filename "$RESULT_FILENAME" \
-    --result-dir /workspace/
+    --result-dir /workspace/ \
+    --trust-remote-code
 
-# After throughput, run evaluation only if RUN_EVAL is true
 if [ "${RUN_EVAL}" = "true" ]; then
     run_eval --framework lm-eval --port "$PORT"
     append_lm_eval_summary
 fi
 
-# Stop GPU monitoring
 stop_gpu_monitor
 set +x
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index e812b0ada..6b34b5e8f 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -119,39 +119,32 @@ dsr1-fp8-mi325x-sglang:
       search-space:
       - { tp: 8, conc-start: 4, conc-end: 64 }
 
-# DeepSeek-V4-Pro FP8 single-node on MI300X (gfx942) via SGLang.
-# EXTRAPOLATED bring-up scaffold (unvalidated on-cluster as of 2026-07).
-# Search-space shape mirrors the same-model MI355X entry (dsv4-fp4-mi355x-sglang):
-# a TP-only low-latency band + a DP-attention + EP throughput band. gfx942 has
-# no native FP4, so this runs FP8 (the ~960GB mixed checkpoint is ~1.05TB in
-# FP8) — which only fits TP8 on 8x192GB, so the MI355X TP4 bands are dropped.
-# Image follows the same-model MI355X's sglang-rocm dated-tag family (mi30x
-# variant); launch script dsv4_fp8_mi300x.sh carries the deepseek_v4 flags +
-# gfx942 AITER infra. debug-runs must confirm the image carries the DeepseekV4
-# class and that --attention-backend dsv4 / --quantization deepseek_v4_fp8
-# work on gfx942 before this merges.
-dsv4-fp8-mi300x-sglang:
-  image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713
+# DeepSeek-V4-Pro FP8 single-node on MI300X (gfx942) via vLLM.
+# EXTRAPOLATED bring-up. sglang was abandoned: on gfx942 (no native FP4) the
+# dsv4 sglang backend's nvfp4 MoE / TileLang-MLA kernels have no gfx942 build
+# (gfx950/MI355X only). vLLM runs the checkpoint in FP8 via --quantization
+# deepseek_v4_fp8 (dequant FP4 MoE -> FP8), the same path the H200 dsv4 vLLM
+# recipe uses. Config mirrors the same-model dsv4-fp4-mi355x-vllm (TP8, conc
+# 4-512); the FP4->FP8 dequant (~1.05TB) fits 8x192GB only at TP8. Launch
+# script dsv4_fp8_mi300x.sh carries the deepseek_v4 + gfx942 AITER flags.
+dsv4-fp8-mi300x-vllm:
+  image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: mi300x
   precision: fp8
-  framework: sglang
+  framework: vllm
   multinode: false
   scenarios:
     fixed-seq-len:
     - isl: 1024
       osl: 1024
       search-space:
-      # Low latency: TP-only (dp-attn off).
-      - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32 }
-      # Throughput: DP-attention + EP8 (mirrors the MI355X tp8 dp-attn band).
-      - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 2048 }
+      - { tp: 8, conc-start: 4, conc-end: 512 }
     - isl: 8192
       osl: 1024
       search-space:
-      - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32 }
-      - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 2048 }
+      - { tp: 8, conc-start: 4, conc-end: 512 }
 
 dsr1-fp8-mi355x-sglang:
   image: lmsysorg/sglang:v0.5.12-rocm700-mi35x
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 8e40f663e..83e8b68c5 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4752,15 +4752,12 @@
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137
 
 - config-keys:
-    - dsv4-fp8-mi300x-sglang
-  description:
-    - "Add DeepSeek-V4-Pro FP8 single-node MI300X SGLang benchmark (new SKU, previously no dsv4 data on gfx942)"
-    - "Extrapolated from the same-model MI355X entry (dsv4-fp4-mi355x-sglang) for search-space shape + deepseek_v4 model flags, and from the same-SKU dsr1-fp8-mi300x-sglang for gfx942 AITER infra; gfx942 has no native FP4 so the FP4-MoE checkpoint runs in FP8"
-    - "No --quantization flag: sglang reads the modelopt quant config from the checkpoint and the AITER MoE path executes it, matching the H200 and MI355X dsv4 sglang recipes (--quantization deepseek_v4_fp8 is vLLM-only and rejected by sglang argparse)"
-    - "SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton (+ SGLANG_USE_ROCM700A=0), mirroring dsv4-fp4-mi355x-sglang: the dsv4 attention backend's TileLang FP8 MLA kernel fails to compile on gfx942 (InjectSoftwarePipeline reorder check) and kills the server at CUDA-graph capture; the unified-KV triton FlashMLA path avoids it"
-    - "--moe-runner-backend triton: the default AITER Composable-Kernel fused-MoE (ck_moe_stage1) has no tuned config for dsv4's FP8 MoE shapes on gfx942 and aborts CUDA-graph capture ('Unsupported kernel config for moe heuristic dispatch'); triton JIT-generates per-shape so it needs no pretuned table (MI355X/gfx950 keeps the CK default, which has gfx950 configs)"
-    - "Search space mirrors MI355X (TP-only low-latency conc 1-32 + DP-attn/EP8 throughput conc 64-2048) but drops the MI355X TP4 bands: the ~960GB mixed checkpoint is ~1.05TB in FP8 and only fits TP8 on 8x192GB"
-    - "Image lmsysorg/sglang-rocm:v0.5.15-rocm720-mi30x-20260713 (mi30x variant of the MI355X sglang-rocm dated-tag family)"
+    - dsv4-fp8-mi300x-vllm
+  description:
+    - "Add DeepSeek-V4-Pro FP8 single-node MI300X vLLM benchmark (new SKU, previously no dsv4 data on gfx942)"
+    - "vLLM, not sglang: on gfx942 (no native FP4) the dsv4 sglang backend's nvfp4 MoE and TileLang-MLA kernels have no gfx942 build (gfx950/MI355X only). vLLM runs the checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), the same path the no-FP4 H200 dsv4 vLLM recipe uses"
+    - "Recipe mirrors the same-model dsv4-fp4-mi355x-vllm (ROCm vLLM dsv4: AITER MoE, deepseek_v4 tokenizer/reasoning-parser, mp executor, FULL_AND_PIECEWISE compile) plus gfx942 AITER infra from minimaxm3-fp8-mi300x-vllm; TP8 conc 4-512 (the FP4->FP8 dequant ~1.05TB fits 8x192GB only at TP8)"
+    - "Image vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (the DSv4-validated ROCm vLLM build)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2194
 
 - config-keys:

From 3f58f40800f3212f54f09277c7af5ec29713fae4 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 14 Jul 2026 21:15:35 +0800
Subject: [PATCH 07/12] =?UTF-8?q?fix(dsv4):=20MI300X=20vLLM=20=E2=80=94=20?=
 =?UTF-8?q?use=20auto=20MoE=20backend=20(AITER=5FMXFP4=5FBF16)=20on=20gfx9?=
 =?UTF-8?q?42?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CI canary root cause (worker traceback): "Mxfp4 MoE backend
'AITER_MXFP4_MXFP4' does not support the deployment configuration ...
QuantKey(u8 ... col=32)". --quantization deepseek_v4_fp8 only dequantizes
the dense/attention weights; the MoE experts stay mxfp4 and go through
vLLM's mxfp4 MoE selector. --moe-backend aiter forced AITER_MXFP4_MXFP4
(W4A4, native mxfp4) which gfx942 has no kernel for.

Drop --moe-backend (default auto): vLLM's select_deepseek_v4_mxfp4_moe_backend
takes its ROCm+DeepseekV4 branch and prefers AITER_MXFP4_BF16 (W4A16 CK,
dequantizes weights — no native FP4) with a TRITON_UNFUSED fallback. MI355X
keeps aiter because gfx950 supports the W4A4 kernel.

中文：MI300X vLLM 去掉 --moe-backend aiter（改用 auto）。deepseek_v4_fp8 仅反
量化稠密/注意力权重，MoE 专家仍为 mxfp4；在 gfx942 上强制 aiter 会选到
AITER_MXFP4_MXFP4（W4A4 原生 mxfp4）而无对应内核。auto 在 ROCm+DeepseekV4
分支优先选 AITER_MXFP4_BF16（W4A16 反量化），并回退 TRITON_UNFUSED。
---
 .../single_node/fixed_seq_len/dsv4_fp8_mi300x.sh      | 11 ++++++++++-
 perf-changelog.yaml                                   |  3 ++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
index 5bedcdf37..07e1c58e6 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
@@ -17,6 +17,16 @@ set -eo pipefail
 #
 # The FP4->FP8 dequant roughly doubles the MoE footprint (~1.05 TB total),
 # which fits 8x192 GB only at TP8, so the sweep is TP8-only.
+#
+# MoE backend is left at auto (NOT --moe-backend aiter). --quantization
+# deepseek_v4_fp8 only handles the dense/attention weights; the MoE experts
+# stay mxfp4 and go through vLLM's mxfp4 MoE selector. On gfx942, forcing
+# aiter selects AITER_MXFP4_MXFP4 (W4A4, native mxfp4) which the gfx942 kernel
+# rejects ("Mxfp4 MoE backend 'AITER_MXFP4_MXFP4' does not support ... QuantKey
+# (u8 ... col=32)"). With auto, vLLM's select_deepseek_v4_mxfp4_moe_backend
+# takes its ROCm+DeepseekV4 branch and prefers AITER_MXFP4_BF16 (W4A16 CK,
+# dequantizes weights — no native FP4), falling back to TRITON_UNFUSED. MI355X
+# keeps --moe-backend aiter because gfx950 supports the W4A4 kernel.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -75,7 +85,6 @@ vllm serve $MODEL --port $PORT \
     --max-model-len "$MAX_MODEL_LEN" \
     --kv-cache-dtype fp8 \
     --trust-remote-code \
-    --moe-backend aiter \
     --tokenizer-mode deepseek_v4 \
     --reasoning-parser deepseek_v4 \
     --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 &
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 83e8b68c5..6de044730 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4756,7 +4756,8 @@
   description:
     - "Add DeepSeek-V4-Pro FP8 single-node MI300X vLLM benchmark (new SKU, previously no dsv4 data on gfx942)"
     - "vLLM, not sglang: on gfx942 (no native FP4) the dsv4 sglang backend's nvfp4 MoE and TileLang-MLA kernels have no gfx942 build (gfx950/MI355X only). vLLM runs the checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), the same path the no-FP4 H200 dsv4 vLLM recipe uses"
-    - "Recipe mirrors the same-model dsv4-fp4-mi355x-vllm (ROCm vLLM dsv4: AITER MoE, deepseek_v4 tokenizer/reasoning-parser, mp executor, FULL_AND_PIECEWISE compile) plus gfx942 AITER infra from minimaxm3-fp8-mi300x-vllm; TP8 conc 4-512 (the FP4->FP8 dequant ~1.05TB fits 8x192GB only at TP8)"
+    - "Recipe mirrors the same-model dsv4-fp4-mi355x-vllm (ROCm vLLM dsv4: deepseek_v4 tokenizer/reasoning-parser, mp executor, FULL_AND_PIECEWISE compile) plus gfx942 AITER infra from minimaxm3-fp8-mi300x-vllm; TP8 conc 4-512 (the FP4->FP8 dequant ~1.05TB fits 8x192GB only at TP8)"
+    - "MoE backend left at auto (unlike MI355X's --moe-backend aiter): deepseek_v4_fp8 only dequantizes dense/attention, the MoE experts stay mxfp4; forcing aiter on gfx942 selects AITER_MXFP4_MXFP4 (W4A4 native-mxfp4) which the gfx942 kernel rejects, while auto's ROCm+DeepseekV4 path prefers AITER_MXFP4_BF16 (W4A16, dequant) with TRITON_UNFUSED fallback"
     - "Image vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (the DSv4-validated ROCm vLLM build)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2194
 

From 117bd8197ad99321e4450ac5d8425c283327b8af Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 00:17:03 +0800
Subject: [PATCH 08/12] fix(dsv4): cap MI300X eval concurrency to gfx942 KV
 budget
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Throughput sweep is green, but the eval job OOM-killed the server mid-gsm8k:
FP8 weights (~131GB/GPU) leave only ~71k tokens of KV on the 192GB MI300X
("Maximum concurrency ... 7.52x" for a 9472-token request), and the eval
drives CONC (128) concurrent requests. Set EVAL_CONCURRENT_REQUESTS=8 to fit
the KV budget. Eval-only (run_eval reads it); throughput jobs use CONC and are
unaffected. MI325X (256GB) has the headroom — its eval already passed — so it
keeps the default.

中文：MI300X 吞吐扫描已通过，但 eval 因 gfx942 KV 太小（FP8 权重占满，仅
~71k tokens KV，9472-token 请求最大并发 7.52x）在 conc 128 下把服务 OOM 杀死。
设置 EVAL_CONCURRENT_REQUESTS=8 匹配 KV 预算（仅影响 eval，不影响吞吐）。
MI325X 显存充足、eval 已通过，保持默认。
---
 benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh | 8 ++++++++
 perf-changelog.yaml                                     | 1 +
 2 files changed, 9 insertions(+)

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
index 07e1c58e6..b921b6054 100755
--- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x.sh
@@ -54,6 +54,14 @@ fi
 export VLLM_ROCM_USE_AITER=1
 export VLLM_ROCM_USE_AITER_MOE=1
 
+# Cap eval concurrency for gfx942's tight KV. FP8 weights (~131GB/GPU) leave
+# only ~71k tokens of KV on the 192GB MI300X ("Maximum concurrency ... 7.52x"
+# for a 9472-token request). The eval defaults to CONC (128) concurrent
+# requests, which OOM-kills the server mid-gsm8k. Cap to the KV budget; this
+# only affects run_eval (throughput jobs use CONC directly). MI325X (256GB)
+# has the headroom and keeps the default.
+export EVAL_CONCURRENT_REQUESTS=8
+
 SERVER_LOG=/workspace/server.log
 
 if [ "${EVAL_ONLY}" = "true" ]; then
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 6de044730..8e56ae692 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4758,6 +4758,7 @@
     - "vLLM, not sglang: on gfx942 (no native FP4) the dsv4 sglang backend's nvfp4 MoE and TileLang-MLA kernels have no gfx942 build (gfx950/MI355X only). vLLM runs the checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), the same path the no-FP4 H200 dsv4 vLLM recipe uses"
     - "Recipe mirrors the same-model dsv4-fp4-mi355x-vllm (ROCm vLLM dsv4: deepseek_v4 tokenizer/reasoning-parser, mp executor, FULL_AND_PIECEWISE compile) plus gfx942 AITER infra from minimaxm3-fp8-mi300x-vllm; TP8 conc 4-512 (the FP4->FP8 dequant ~1.05TB fits 8x192GB only at TP8)"
     - "MoE backend left at auto (unlike MI355X's --moe-backend aiter): deepseek_v4_fp8 only dequantizes dense/attention, the MoE experts stay mxfp4; forcing aiter on gfx942 selects AITER_MXFP4_MXFP4 (W4A4 native-mxfp4) which the gfx942 kernel rejects, while auto's ROCm+DeepseekV4 path prefers AITER_MXFP4_BF16 (W4A16, dequant) with TRITON_UNFUSED fallback"
+    - "EVAL_CONCURRENT_REQUESTS=8: FP8 weights (~131GB/GPU) leave only ~71k tokens of KV on 8x192GB (max 7.52x concurrency for a 9472-token request); the eval default of CONC (128) concurrent requests OOM-kills the server mid-gsm8k, so eval concurrency is capped to the KV budget (throughput unaffected; MI325X's 256GB keeps the default)"
     - "Image vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (the DSv4-validated ROCm vLLM build)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2194
 

From 4a5b167ef605776af3363b781b1d8327a2cd589f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 01:01:09 +0800
Subject: [PATCH 09/12] feat(dsv4): add MI300X vLLM MTP spec-decode variant
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

dsv4-fp8-mi300x-vllm-mtp mirrors the working normal recipe (auto MoE ->
AITER_MXFP4_BF16, deepseek_v4_fp8, EVAL_CONCURRENT_REQUESTS=8) and adds
DeepSeek-V4 built-in MTP via --speculative-config
{"method":"mtp","num_speculative_tokens":2}, with --dsv4 chat-template
encoding for run_benchmark_serving (required for meaningful MTP acceptance).
Routes to dsv4_fp8_mi300x_mtp.sh via the launcher's spec-decoding=mtp suffix.

中文：新增 MI300X vLLM MTP（投机解码）变体，在正常配方基础上加入
--speculative-config mtp 与 --dsv4 chat-template 编码。
---
 .../fixed_seq_len/dsv4_fp8_mi300x_mtp.sh      | 133 ++++++++++++++++++
 configs/amd-master.yaml                       |  23 +++
 perf-changelog.yaml                           |   8 ++
 3 files changed, 164 insertions(+)
 create mode 100755 benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x_mtp.sh

diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x_mtp.sh
new file mode 100755
index 000000000..0f71361f6
--- /dev/null
+++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_mi300x_mtp.sh
@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# DeepSeek-V4-Pro FP8 single-node on MI300X (gfx942) via vLLM, MTP variant.
+#
+# MTP sibling of dsv4_fp8_mi300x.sh: adds --speculative-config
+# '{"method":"mtp","num_speculative_tokens":2}' (DeepSeek-V4 built-in MTP)
+# and --dsv4 chat-template encoding for run_benchmark_serving (EAGLE/MTP is
+# trained on chat-formatted inputs).
+#
+# EXTRAPOLATED bring-up recipe. The sglang path was abandoned: on gfx942
+# (no native FP4) the dsv4 sglang backend's nvfp4 MoE / TileLang-MLA kernels
+# have no gfx942 equivalents (they exist only for gfx950/MI355X). vLLM instead
+# runs the checkpoint in FP8 via --quantization deepseek_v4_fp8, which
+# dequantizes the FP4 MoE experts to FP8 — the same path the H200 dsv4 vLLM
+# recipe uses (H200 is also a no-FP4 SKU). Derived from:
+#   * same model + framework + AMD family: dsv4_fp4_mi355x_vllm.sh (ROCm vLLM
+#     dsv4 structure: AITER MoE, deepseek_v4 tokenizer/parser, mp executor,
+#     FULL_AND_PIECEWISE compile)
+#   * same model, FP8 path: dsv4_fp8_h200.sh (--quantization deepseek_v4_fp8)
+#   * same SKU, different model: minimaxm3_fp8_mi300x.sh (gfx942 vLLM/AITER)
+#
+# The FP4->FP8 dequant roughly doubles the MoE footprint (~1.05 TB total),
+# which fits 8x192 GB only at TP8, so the sweep is TP8-only.
+#
+# MoE backend is left at auto (NOT --moe-backend aiter). --quantization
+# deepseek_v4_fp8 only handles the dense/attention weights; the MoE experts
+# stay mxfp4 and go through vLLM's mxfp4 MoE selector. On gfx942, forcing
+# aiter selects AITER_MXFP4_MXFP4 (W4A4, native mxfp4) which the gfx942 kernel
+# rejects ("Mxfp4 MoE backend 'AITER_MXFP4_MXFP4' does not support ... QuantKey
+# (u8 ... col=32)"). With auto, vLLM's select_deepseek_v4_mxfp4_moe_backend
+# takes its ROCm+DeepseekV4 branch and prefers AITER_MXFP4_BF16 (W4A16 CK,
+# dequantizes weights — no native FP4), falling back to TRITON_UNFUSED. MI355X
+# keeps --moe-backend aiter because gfx950 supports the W4A4 kernel.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars \
+    MODEL \
+    TP \
+    DP_ATTENTION \
+    CONC \
+    ISL \
+    OSL \
+    MAX_MODEL_LEN \
+    RANDOM_RANGE_RATIO \
+    RESULT_FILENAME
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+  echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi
+
+if [ -n "$ROCR_VISIBLE_DEVICES" ]; then
+    export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
+fi
+
+export VLLM_ROCM_USE_AITER=1
+export VLLM_ROCM_USE_AITER_MOE=1
+
+# Cap eval concurrency for gfx942's tight KV. FP8 weights (~131GB/GPU) leave
+# only ~71k tokens of KV on the 192GB MI300X ("Maximum concurrency ... 7.52x"
+# for a 9472-token request). The eval defaults to CONC (128) concurrent
+# requests, which OOM-kills the server mid-gsm8k. Cap to the KV budget; this
+# only affects run_eval (throughput jobs use CONC directly). MI325X (256GB)
+# has the headroom and keeps the default.
+export EVAL_CONCURRENT_REQUESTS=8
+
+SERVER_LOG=/workspace/server.log
+
+if [ "${EVAL_ONLY}" = "true" ]; then
+    setup_eval_context
+    MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN"
+fi
+
+start_gpu_monitor
+
+PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
+if [ "${DP_ATTENTION}" = "true" ]; then
+    PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
+fi
+
+EP_ARGS=()
+if [ "${EP_SIZE:-1}" -gt 1 ]; then
+    EP_ARGS=(--enable-expert-parallel)
+fi
+
+# Use 2 speculative tokens (matches dsv4_fp4_mi355x_vllm_mtp.sh).
+NUM_SPEC_TOKENS=2
+
+set -x
+vllm serve $MODEL --port $PORT \
+    "${PARALLEL_ARGS[@]}" \
+    "${EP_ARGS[@]}" \
+    --quantization deepseek_v4_fp8 \
+    --async-scheduling \
+    --no-enable-prefix-caching \
+    --distributed-executor-backend mp \
+    --gpu-memory-utilization 0.9 \
+    --max-model-len "$MAX_MODEL_LEN" \
+    --kv-cache-dtype fp8 \
+    --trust-remote-code \
+    --tokenizer-mode deepseek_v4 \
+    --reasoning-parser deepseek_v4 \
+    --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \
+    --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 &
+
+SERVER_PID=$!
+
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+run_benchmark_serving \
+    --model "$MODEL" \
+    --port "$PORT" \
+    --backend vllm \
+    --input-len "$ISL" \
+    --output-len "$OSL" \
+    --random-range-ratio "$RANDOM_RANGE_RATIO" \
+    --num-prompts "$((CONC * 10))" \
+    --max-concurrency "$CONC" \
+    --result-filename "$RESULT_FILENAME" \
+    --result-dir /workspace/ \
+    --trust-remote-code \
+    --dsv4
+
+if [ "${RUN_EVAL}" = "true" ]; then
+    run_eval --framework lm-eval --port "$PORT"
+    append_lm_eval_summary
+fi
+
+stop_gpu_monitor
+set +x
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index 6b34b5e8f..d8ae62d4f 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -146,6 +146,29 @@ dsv4-fp8-mi300x-vllm:
       search-space:
       - { tp: 8, conc-start: 4, conc-end: 512 }
 
+# MTP variant of dsv4-fp8-mi300x-vllm. Mirrors the base recipe and adds
+# DeepSeek-V4 built-in MTP via --speculative-config (num_speculative_tokens=2),
+# routing to dsv4_fp8_mi300x_mtp.sh; benchmark uses --dsv4 chat-template
+# encoding (required for meaningful MTP acceptance).
+dsv4-fp8-mi300x-vllm-mtp:
+  image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: mi300x
+  precision: fp8
+  framework: vllm
+  multinode: false
+  scenarios:
+    fixed-seq-len:
+    - isl: 1024
+      osl: 1024
+      search-space:
+      - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp }
+    - isl: 8192
+      osl: 1024
+      search-space:
+      - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp }
+
 dsr1-fp8-mi355x-sglang:
   image: lmsysorg/sglang:v0.5.12-rocm700-mi35x
   model: deepseek-ai/DeepSeek-R1-0528
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 8e56ae692..3bd5626b1 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4776,3 +4776,11 @@
     - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708"
     - "Clean the export envs"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198
+
+- config-keys:
+    - dsv4-fp8-mi300x-vllm-mtp
+  description:
+    - "Add DeepSeek-V4-Pro FP8 MI300X vLLM MTP (speculative-decoding) variant, mirroring dsv4-fp8-mi300x-vllm plus --speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":2} (DeepSeek-V4 built-in MTP)"
+    - "run_benchmark_serving uses --dsv4 chat-template encoding, required for meaningful MTP acceptance rate (EAGLE/MTP is trained on chat-formatted inputs)"
+    - "Same gfx942 auto-MoE (AITER_MXFP4_BF16) + EVAL_CONCURRENT_REQUESTS=8 KV cap as the base recipe; routes to dsv4_fp8_mi300x_mtp.sh via the launcher's spec-decoding=mtp suffix"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2194

From 5a0a5b5107ae87d91e47e1ee6527a83819bebeb0 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 01:24:47 +0800
Subject: [PATCH 10/12] fix(dsv4): rebase MI300X perf-changelog onto
 origin/main (append-only)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

origin/main advanced (H200 refresh #2191 merged); reset perf-changelog.yaml
to origin/main and re-append the dsv4-fp8-mi300x-vllm and
dsv4-fp8-mi300x-vllm-mtp entries so check-changelog's append-only rule passes.

中文：将 MI300X 分支 perf-changelog 重置到 origin/main 并重新追加其条目。
---
 perf-changelog.yaml | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 3bd5626b1..55228a17c 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4752,15 +4752,20 @@
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137
 
 - config-keys:
-    - dsv4-fp8-mi300x-vllm
+    - dsv4-fp8-h200-vllm
+    - dsv4-fp8-h200-vllm-mtp
   description:
-    - "Add DeepSeek-V4-Pro FP8 single-node MI300X vLLM benchmark (new SKU, previously no dsv4 data on gfx942)"
-    - "vLLM, not sglang: on gfx942 (no native FP4) the dsv4 sglang backend's nvfp4 MoE and TileLang-MLA kernels have no gfx942 build (gfx950/MI355X only). vLLM runs the checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), the same path the no-FP4 H200 dsv4 vLLM recipe uses"
-    - "Recipe mirrors the same-model dsv4-fp4-mi355x-vllm (ROCm vLLM dsv4: deepseek_v4 tokenizer/reasoning-parser, mp executor, FULL_AND_PIECEWISE compile) plus gfx942 AITER infra from minimaxm3-fp8-mi300x-vllm; TP8 conc 4-512 (the FP4->FP8 dequant ~1.05TB fits 8x192GB only at TP8)"
-    - "MoE backend left at auto (unlike MI355X's --moe-backend aiter): deepseek_v4_fp8 only dequantizes dense/attention, the MoE experts stay mxfp4; forcing aiter on gfx942 selects AITER_MXFP4_MXFP4 (W4A4 native-mxfp4) which the gfx942 kernel rejects, while auto's ROCm+DeepseekV4 path prefers AITER_MXFP4_BF16 (W4A16, dequant) with TRITON_UNFUSED fallback"
-    - "EVAL_CONCURRENT_REQUESTS=8: FP8 weights (~131GB/GPU) leave only ~71k tokens of KV on 8x192GB (max 7.52x concurrency for a 9472-token request); the eval default of CONC (128) concurrent requests OOM-kills the server mid-gsm8k, so eval concurrency is capped to the KV budget (throughput unaffected; MI325X's 256GB keeps the default)"
-    - "Image vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (the DSv4-validated ROCm vLLM build)"
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2194
+    - "Bump vLLM image from v0.21.0 to v0.25.0 for DeepSeek-V4-Pro FP8 on H200, matching the B200/B300 dsv4 vLLM bump (#2169)"
+    - "Refresh stale H200 dsv4 submissions (last run 2026-05-21)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2191
+
+- config-keys:
+    - dsv4-fp8-h200-sglang
+    - dsv4-fp8-h200-sglang-mtp
+  description:
+    - "Bump the pinned lmsysorg/sglang:deepseek-v4-hopper digest from the 2026-05-02 push (7f19c6dc) to the current 2026-05-13 push (1bf5d508)"
+    - "Refresh stale H200 dsv4 SGLang submissions (last run 2026-05-04)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2191
 
 - config-keys:
     - dsv4-fp4-mi355x-sglang
@@ -4777,6 +4782,17 @@
     - "Clean the export envs"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198
 
+- config-keys:
+    - dsv4-fp8-mi300x-vllm
+  description:
+    - "Add DeepSeek-V4-Pro FP8 single-node MI300X vLLM benchmark (new SKU, previously no dsv4 data on gfx942)"
+    - "vLLM, not sglang: on gfx942 (no native FP4) the dsv4 sglang backend's nvfp4 MoE and TileLang-MLA kernels have no gfx942 build (gfx950/MI355X only). vLLM runs the checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), the same path the no-FP4 H200 dsv4 vLLM recipe uses"
+    - "Recipe mirrors the same-model dsv4-fp4-mi355x-vllm (ROCm vLLM dsv4: deepseek_v4 tokenizer/reasoning-parser, mp executor, FULL_AND_PIECEWISE compile) plus gfx942 AITER infra from minimaxm3-fp8-mi300x-vllm; TP8 conc 4-512 (the FP4->FP8 dequant ~1.05TB fits 8x192GB only at TP8)"
+    - "MoE backend left at auto (unlike MI355X's --moe-backend aiter): deepseek_v4_fp8 only dequantizes dense/attention, the MoE experts stay mxfp4; forcing aiter on gfx942 selects AITER_MXFP4_MXFP4 (W4A4 native-mxfp4) which the gfx942 kernel rejects, while auto's ROCm+DeepseekV4 path prefers AITER_MXFP4_BF16 (W4A16, dequant) with TRITON_UNFUSED fallback"
+    - "EVAL_CONCURRENT_REQUESTS=8: FP8 weights (~131GB/GPU) leave only ~71k tokens of KV on 8x192GB (max 7.52x concurrency for a 9472-token request); the eval default of CONC (128) concurrent requests OOM-kills the server mid-gsm8k, so eval concurrency is capped to the KV budget (throughput unaffected; MI325X's 256GB keeps the default)"
+    - "Image vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa (the DSv4-validated ROCm vLLM build)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2194
+
 - config-keys:
     - dsv4-fp8-mi300x-vllm-mtp
   description:

From 6d49f8473b15945cd2a62ad38471d5a18862d2ae Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 04:26:00 +0800
Subject: [PATCH 11/12] fix(dsv4): cap MI300X 8k1k conc at 256

8k1k KV fits only ~20x concurrency; conc512 oversubscribed -> request timeouts
(MTP c512 8k1k: 55.8% failures on both AMD SKUs). conc256 passed for normal and
MTP; cap 8k1k conc-end at 256 (1k1k holds at 512). Eval clamp already present.
---
 configs/amd-master.yaml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index d8ae62d4f..cfcd8ea11 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -144,7 +144,9 @@ dsv4-fp8-mi300x-vllm:
     - isl: 8192
       osl: 1024
       search-space:
-      - { tp: 8, conc-start: 4, conc-end: 512 }
+      # 8k1k KV fits only ~20x concurrency (9472-token requests), so conc512 is
+      # ~25x oversubscribed -> request timeouts. conc256 is proven green; cap here.
+      - { tp: 8, conc-start: 4, conc-end: 256 }
 
 # MTP variant of dsv4-fp8-mi300x-vllm. Mirrors the base recipe and adds
 # DeepSeek-V4 built-in MTP via --speculative-config (num_speculative_tokens=2),
@@ -167,7 +169,9 @@ dsv4-fp8-mi300x-vllm-mtp:
     - isl: 8192
       osl: 1024
       search-space:
-      - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp }
+      # 8k1k MTP at conc512 failed (55.8% request failures, KV ~20x oversubscribed);
+      # conc256 passed on both MI300X/MI325X. Cap 8k1k MTP here (1k1k holds at 512).
+      - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp }
 
 dsr1-fp8-mi355x-sglang:
   image: lmsysorg/sglang:v0.5.12-rocm700-mi35x

From 2340fd4b1bfd3f1bb7a5cd40bf9092c4018ef2cb Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 15 Jul 2026 06:46:40 +0800
Subject: [PATCH 12/12] fix(dsv4): lower MI300X MTP 8k1k conc cap 256->128

MTP 8k1k conc256 failed (19.2% request failures) -- the MTP draft model
shrinks KV below normal's, so 256 oversubscribes the ~20x ceiling. conc128
passed cleanly. Normal 8k1k holds 256 (passed), 1k1k holds 512.
---
 configs/amd-master.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index cfcd8ea11..0d6577d59 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -169,9 +169,10 @@ dsv4-fp8-mi300x-vllm-mtp:
     - isl: 8192
       osl: 1024
       search-space:
-      # 8k1k MTP at conc512 failed (55.8% request failures, KV ~20x oversubscribed);
-      # conc256 passed on both MI300X/MI325X. Cap 8k1k MTP here (1k1k holds at 512).
-      - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp }
+      # 8k1k MTP KV is tighter than normal (draft model): conc256 failed here
+      # (19.2% req failures) and conc512 was 55.8%; conc128 passed cleanly.
+      # Cap 8k1k MTP at 128 (normal holds 256, 1k1k holds 512).
+      - { tp: 8, conc-start: 4, conc-end: 128, spec-decoding: mtp }
 
 dsr1-fp8-mi355x-sglang:
   image: lmsysorg/sglang:v0.5.12-rocm700-mi35x