[WIP] Update vllm B300 Minimax M3

benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh

@@ -3,7 +3,14 @@
 # MiniMax-M3 NVFP4 B300 single-node vLLM recipe.
 # Same shape as minimaxm3_fp8_b300.sh but uses the nvidia/MiniMax-M3-NVFP4
 # checkpoint. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm PR #46380) is
-# baked into the perf container image, so no runtime patch is needed.
+# baked into the perf container image.
+#
+# At runtime the recipe swaps the image's FlashInfer for a pinned nightly with
+# the upstream SM100 low-M MXFP8 split-K kernel (flashinfer-ai/flashinfer#3847),
+# the distributed AutoTuner synchronization API (#3187), and the non-Tensor
+# guard (#3918). It backports the still-unmerged #3912 memory fix and patches
+# vLLM to opt in to synchronized distributed tuning. vLLM PR #48268 supplies
+# the per-op AutoTuner skip control used below.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -19,6 +26,46 @@ check_env_vars \
     RANDOM_RANGE_RATIO \
     RESULT_FILENAME
 
+# --- FlashInfer nightly + targeted runtime patches --------------------------
+FLASHINFER_VERSION=0.6.15.dev20260712
+FLASHINFER_NIGHTLY_TAG=nightly-v0.6.15-20260712
+FLASHINFER_RELEASE_URL="https://github.com/flashinfer-ai/flashinfer/releases/download/${FLASHINFER_NIGHTLY_TAG}"
+
+python3 -m pip uninstall -y flashinfer-python flashinfer-cubin flashinfer-jit-cache
+
+python3 -m pip install \
+    "${FLASHINFER_RELEASE_URL}/flashinfer_python-${FLASHINFER_VERSION}-py3-none-any.whl" \
+    "${FLASHINFER_RELEASE_URL}/flashinfer_cubin-${FLASHINFER_VERSION}-py3-none-any.whl" \
+    "${FLASHINFER_RELEASE_URL}/flashinfer_jit_cache-${FLASHINFER_VERSION}+cu130-cp39-abi3-manylinux_2_28_$(uname -m).whl" \
+    || { echo "FlashInfer nightly install failed" >&2; exit 1; }
+
+if ! command -v patch >/dev/null 2>&1; then
+    apt-get update -y && apt-get install -y --no-install-recommends patch \
+        || { echo "Failed to install patch(1)" >&2; exit 1; }
+fi
+SITE_PACKAGES=$(dirname "$(python3 -c "import importlib.util; print(importlib.util.find_spec('flashinfer').submodule_search_locations[0])")") \
+    || { echo "Could not locate the installed flashinfer package" >&2; exit 1; }
+
+# Backport the runtime portion of flashinfer-ai/flashinfer#3912. Caching the
+# packed top-k initializer preserves its identity across tuning calls and avoids
+# retaining a fresh closure in the AutoTuner cache for every invocation.
+AUTOTUNER_MEMORY_PATCH="$(dirname "$0")/patches/flashinfer-pr-3912.patch"
+patch --dry-run -p1 -d "${SITE_PACKAGES}" < "${AUTOTUNER_MEMORY_PATCH}" >/dev/null \
+    || { echo "FlashInfer PR #3912 patch does not apply" >&2; exit 1; }
+patch -p1 -d "${SITE_PACKAGES}" < "${AUTOTUNER_MEMORY_PATCH}" \
+    || { echo "FlashInfer PR #3912 patch failed" >&2; exit 1; }
+
+# FlashInfer #3187 exposes distributed tactic synchronization as an opt-in API.
+# Wire vLLM's multi-rank warmup to the existing gloo world group so every rank
+# reduces the same profile timings before selecting a tactic.
+VLLM_AUTOTUNER_GROUP_PATCH="$(dirname "$0")/patches/vllm-flashinfer-autotune-process-group.patch"
+patch --dry-run -p1 -d "${SITE_PACKAGES}" < "${VLLM_AUTOTUNER_GROUP_PATCH}" >/dev/null \
+    || { echo "vLLM FlashInfer AutoTuner process-group patch does not apply" >&2; exit 1; }
+patch -p1 -d "${SITE_PACKAGES}" < "${VLLM_AUTOTUNER_GROUP_PATCH}" \
+    || { echo "vLLM FlashInfer AutoTuner process-group patch failed" >&2; exit 1; }
+
+# -----------------------------------------------------------------------------
+
 if [[ -n "${MODEL_PATH:-}" ]]; then
     if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
         hf download "$MODEL" --local-dir "$MODEL_PATH"
@@ -39,6 +86,8 @@ SERVER_LOG=/workspace/server.log
 export VLLM_ENGINE_READY_TIMEOUT_S=3600
 export VLLM_FLOAT32_MATMUL_PRECISION=high
 export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm
+export VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS='flashinfer::trtllm_fp4_block_scale_moe'
+export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=300
 
 if [ "${DP_ATTENTION}" = "true" ]; then
   PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel"
@@ -52,19 +101,30 @@ if [ "${EVAL_ONLY}" = "true" ]; then
     setup_eval_context
     MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN"
 fi
+
+GPU_MEMORY_UTILIZATION=0.95
+TP1_EXTRA_ARGS=""
+if [ "$TP" -eq 1 ]; then
+    GPU_MEMORY_UTILIZATION=0.97
+    TP1_EXTRA_ARGS="--max-num-seqs 16 --compilation_config.cudagraph_mode FULL_DECODE_ONLY"
+fi
+
 start_gpu_monitor
 
 set -x
 vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \
 $PARALLEL_ARGS \
---gpu-memory-utilization 0.95 \
+$TP1_EXTRA_ARGS \
+--attention_config.indexer_kv_dtype fp8 \
+--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
 --max-model-len $MAX_MODEL_LEN \
 --kv-cache-dtype fp8 \
 --block-size 128 \
 --language-model-only \
 --max-cudagraph-capture-size 2048 \
 --max-num-batched-tokens "$((ISL * 2 ))" \
 --stream-interval 20 --no-enable-prefix-caching \
+--enable-flashinfer-autotune \
 --trust-remote-code > $SERVER_LOG 2>&1 &
 
 SERVER_PID=$!

benchmarks/single_node/fixed_seq_len/patches/flashinfer-pr-3912.patch

@@ -0,0 +1,67 @@
+diff --git a/flashinfer/fused_moe/core.py b/flashinfer/fused_moe/core.py
+--- a/flashinfer/fused_moe/core.py
++++ b/flashinfer/fused_moe/core.py
+@@ -86,6 +86,31 @@ from .utils import (
+ )
+
+
++@functools.cache
++def _moe_topk_ids_init(num_experts: int):
++    """Return a packed-topk-ids initializer for a given expert count. Cached for
++    object identity preservation.
++    """
++
++    def _init(
++        shapes: tuple[int, ...],
++        dtype: torch.dtype,
++        device: torch.device,
++    ) -> torch.Tensor:
++        expert_ids = make_random_topk_ids(
++            num_experts=num_experts,
++            num_tokens=math.prod(shapes[:-1]),
++            top_k=shapes[-1],
++            device=device,
++        ).view(shapes)
++        expert_weights = torch.ones(shapes, dtype=torch.bfloat16, device=device).view(
++            torch.int16
++        )
++        return (expert_ids << 16) | expert_weights
++
++    return _init
++
++
+ # Routing input modes for FusedMoE launcher
+ # Please keep this in sync with the counterpart defined in csrc/trtllm_fused_moe_kernel_launcher.cu
+ class RoutingInputMode(IntEnum):
+@@ -1250,30 +1275,14 @@ class FusedMoE:
+                 **kwargs: Extra TuningConfig kwargs (e.g. use_cold_l2_cache).
+             """
+
+-            def _init_packed_topk_ids(
+-                shapes: tuple[int, ...],
+-                dtype: torch.dtype,
+-                device: torch.device,
+-            ) -> torch.Tensor:
+-                expert_ids = make_random_topk_ids(
+-                    num_experts=self.num_experts,
+-                    num_tokens=math.prod(shapes[:-1]),
+-                    top_k=shapes[-1],
+-                    device=device,
+-                ).view(shapes)
+-                expert_weights = torch.ones(
+-                    shapes, dtype=torch.bfloat16, device=device
+-                ).view(torch.int16)
+-                return (expert_ids << 16) | expert_weights
+-
+             spec = {
+                 "output": autotuner_initializer_empty,
+                 "hidden_states": autotuner_initializer_randn,
+             }
+             if moe_inputs.routing_logits is not None:
+                 spec["routing_logits"] = autotuner_initializer_rand
+             if moe_inputs.topk_ids is not None:
+-                spec["topk_ids"] = _init_packed_topk_ids
++                spec["topk_ids"] = _moe_topk_ids_init(self.num_experts)
+             if moe_inputs.expert_weights is not None:
+                 spec["expert_weights"] = autotuner_initializer_ones
+             if moe_inputs.hidden_states_scale is not None:

benchmarks/single_node/fixed_seq_len/patches/vllm-flashinfer-autotune-process-group.patch

@@ -0,0 +1,27 @@
+diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py
+--- a/vllm/model_executor/warmup/kernel_warmup.py
++++ b/vllm/model_executor/warmup/kernel_warmup.py
+@@ -178,9 +178,16 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
+     if not use_persistent_cache:
+-        with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs):
+-            runner._dummy_run(
+-                num_tokens=runner.scheduler_config.max_num_batched_tokens,
+-                skip_eplb=True,
+-                is_profile=True,
+-            )
+-        get_world_group().barrier()
++        from flashinfer.autotuner import set_autotune_process_group
++
++        world = get_world_group()
++        set_autotune_process_group(world.cpu_group)
++        try:
++            with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs):
++                runner._dummy_run(
++                    num_tokens=runner.scheduler_config.max_num_batched_tokens,
++                    skip_eplb=True,
++                    is_profile=True,
++                )
++        finally:
++            set_autotune_process_group(None)
++        world.barrier()
+         return

configs/nvidia-master.yaml

@@ -13697,7 +13697,7 @@ minimaxm3-fp8-b300-vllm:
 # weights are pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added to
 # the STAGED_MODELS allow-list in launch_b300-nv.sh).
 minimaxm3-fp4-b300-vllm:
-  image: vllm/vllm-openai:nightly-93d8f834dd8acf33eb0e2a75b2711b628cb6e226
+  image: vllm/vllm-openai:nightly-3d99b0499aff7ce3b91942b0385558dab4eecf76
   model: nvidia/MiniMax-M3-NVFP4
   model-prefix: minimaxm3
   runner: b300
@@ -13706,22 +13706,13 @@ minimaxm3-fp4-b300-vllm:
   multinode: false
   scenarios:
     fixed-seq-len:
-    - isl: 1024
-      osl: 1024
-      search-space:
-      - { tp: 8, conc-start: 1, conc-end: 2 }
-      - { tp: 4, conc-start: 1, conc-end: 64 }
-      - { tp: 2, conc-start: 4, conc-end: 256 }
-      - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 }
-      - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 }
-      - { tp: 2, ep: 2, dp-attn: true, conc-start: 4096, conc-end: 4096 }
     - isl: 8192
       osl: 1024
       search-space:
+      - { tp: 1, conc-start: 1, conc-end: 16 }
       - { tp: 8, conc-start: 1, conc-end: 2 }
       - { tp: 4, conc-start: 1, conc-end: 2 }
       - { tp: 2, conc-start: 4, conc-end: 256 }
-      - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 }
 
 # EAGLE3 speculative-decoding (spec-decoding: mtp) variant of MiniMax-M3 NVFP4
 # (nvidia/MiniMax-M3-NVFP4) B300 single-node vLLM, pairing the target with the

perf-changelog.yaml

@@ -4781,3 +4781,38 @@
     - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708"
     - "Clean the export envs"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198
+
+- config-keys:
+    - minimaxm3-fp4-b300-vllm
+  description:
+    - "Update the vLLM image to nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9"
+    - "Install FlashInfer 0.6.15.dev20260712"
+    - "Use the upstream SM100 low-M MXFP8 split-K implementation included in FlashInfer 0.6.15"
+    - "Use the upstream FlashInfer PR #3187 distributed AutoTuner synchronization API and PR #3918 non-Tensor guard included in the 0712 nightly"
+    - "Apply the FlashInfer PR #3912 AutoTuner cache and memory fix via a local runtime patch"
+    - "Keep the upstream FlashInfer PR #3582, PR #3687, and PR #3745 runtime changes"
+    - "Patch vLLM to synchronize distributed FlashInfer tactic selection over its gloo world group and re-enable AutoTuner"
+    - "Set VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=300 to fail GPU hangs faster"
+    - "Use the explicit FlashInfer TRT-LLM all-reduce backend override"
+    - "Remove MiniMax 1k1k and 8k1k DEP2 points from this performance bisect"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2124
+
+- config-keys:
+    - minimaxm3-fp4-b300-vllm
+  description:
+    - "Update the vLLM image to nightly-3d99b0499aff7ce3b91942b0385558dab4eecf76, the earliest nightly containing vLLM PR #48268"
+    - "Skip FlashInfer AutoTuner profiling for flashinfer::trtllm_fp4_block_scale_moe while retaining synchronized tuning for the remaining operations"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2124
+
+- config-keys:
+    - minimaxm3-fp4-b300-vllm
+  description:
+    - "Add an 8k1k TP1 diagnostic sweep at concurrency 1-16"
+    - "Use vLLM GPU memory utilization 0.97 for TP1 while retaining 0.95 for all other TP sizes"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2124
+
+- config-keys:
+    - minimaxm3-fp4-b300-vllm
+  description:
+    - "Set --max-num-seqs 16 and FULL_DECODE_ONLY CUDA Graph mode for TP1 only"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2124