diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh index f91419edb7..9e4c80639b 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh @@ -3,7 +3,14 @@ # MiniMax-M3 NVFP4 B300 single-node vLLM recipe. # Same shape as minimaxm3_fp8_b300.sh but uses the nvidia/MiniMax-M3-NVFP4 # checkpoint. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm PR #46380) is -# baked into the perf container image, so no runtime patch is needed. +# baked into the perf container image. +# +# At runtime the recipe swaps the image's FlashInfer for a pinned nightly with +# the upstream SM100 low-M MXFP8 split-K kernel (flashinfer-ai/flashinfer#3847), +# the distributed AutoTuner synchronization API (#3187), and the non-Tensor +# guard (#3918). It backports the still-unmerged #3912 memory fix and patches +# vLLM to opt in to synchronized distributed tuning. vLLM PR #48268 supplies +# the per-op AutoTuner skip control used below. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -19,6 +26,46 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# --- FlashInfer nightly + targeted runtime patches -------------------------- +FLASHINFER_VERSION=0.6.15.dev20260712 +FLASHINFER_NIGHTLY_TAG=nightly-v0.6.15-20260712 +FLASHINFER_RELEASE_URL="https://github.com/flashinfer-ai/flashinfer/releases/download/${FLASHINFER_NIGHTLY_TAG}" + +python3 -m pip uninstall -y flashinfer-python flashinfer-cubin flashinfer-jit-cache + +python3 -m pip install \ + "${FLASHINFER_RELEASE_URL}/flashinfer_python-${FLASHINFER_VERSION}-py3-none-any.whl" \ + "${FLASHINFER_RELEASE_URL}/flashinfer_cubin-${FLASHINFER_VERSION}-py3-none-any.whl" \ + "${FLASHINFER_RELEASE_URL}/flashinfer_jit_cache-${FLASHINFER_VERSION}+cu130-cp39-abi3-manylinux_2_28_$(uname -m).whl" \ + || { echo "FlashInfer nightly install failed" >&2; exit 1; } + +if ! command -v patch >/dev/null 2>&1; then + apt-get update -y && apt-get install -y --no-install-recommends patch \ + || { echo "Failed to install patch(1)" >&2; exit 1; } +fi +SITE_PACKAGES=$(dirname "$(python3 -c "import importlib.util; print(importlib.util.find_spec('flashinfer').submodule_search_locations[0])")") \ + || { echo "Could not locate the installed flashinfer package" >&2; exit 1; } + +# Backport the runtime portion of flashinfer-ai/flashinfer#3912. Caching the +# packed top-k initializer preserves its identity across tuning calls and avoids +# retaining a fresh closure in the AutoTuner cache for every invocation. +AUTOTUNER_MEMORY_PATCH="$(dirname "$0")/patches/flashinfer-pr-3912.patch" +patch --dry-run -p1 -d "${SITE_PACKAGES}" < "${AUTOTUNER_MEMORY_PATCH}" >/dev/null \ + || { echo "FlashInfer PR #3912 patch does not apply" >&2; exit 1; } +patch -p1 -d "${SITE_PACKAGES}" < "${AUTOTUNER_MEMORY_PATCH}" \ + || { echo "FlashInfer PR #3912 patch failed" >&2; exit 1; } + +# FlashInfer #3187 exposes distributed tactic synchronization as an opt-in API. +# Wire vLLM's multi-rank warmup to the existing gloo world group so every rank +# reduces the same profile timings before selecting a tactic. +VLLM_AUTOTUNER_GROUP_PATCH="$(dirname "$0")/patches/vllm-flashinfer-autotune-process-group.patch" +patch --dry-run -p1 -d "${SITE_PACKAGES}" < "${VLLM_AUTOTUNER_GROUP_PATCH}" >/dev/null \ + || { echo "vLLM FlashInfer AutoTuner process-group patch does not apply" >&2; exit 1; } +patch -p1 -d "${SITE_PACKAGES}" < "${VLLM_AUTOTUNER_GROUP_PATCH}" \ + || { echo "vLLM FlashInfer AutoTuner process-group patch failed" >&2; exit 1; } + +# ----------------------------------------------------------------------------- + if [[ -n "${MODEL_PATH:-}" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" @@ -39,6 +86,8 @@ SERVER_LOG=/workspace/server.log export VLLM_ENGINE_READY_TIMEOUT_S=3600 export VLLM_FLOAT32_MATMUL_PRECISION=high export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm +export VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS='flashinfer::trtllm_fp4_block_scale_moe' +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=300 if [ "${DP_ATTENTION}" = "true" ]; then PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" @@ -52,12 +101,22 @@ if [ "${EVAL_ONLY}" = "true" ]; then setup_eval_context MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" fi + +GPU_MEMORY_UTILIZATION=0.95 +TP1_EXTRA_ARGS="" +if [ "$TP" -eq 1 ]; then + GPU_MEMORY_UTILIZATION=0.97 + TP1_EXTRA_ARGS="--max-num-seqs 16 --compilation_config.cudagraph_mode FULL_DECODE_ONLY" +fi + start_gpu_monitor set -x vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ $PARALLEL_ARGS \ ---gpu-memory-utilization 0.95 \ +$TP1_EXTRA_ARGS \ +--attention_config.indexer_kv_dtype fp8 \ +--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \ --max-model-len $MAX_MODEL_LEN \ --kv-cache-dtype fp8 \ --block-size 128 \ @@ -65,6 +124,7 @@ $PARALLEL_ARGS \ --max-cudagraph-capture-size 2048 \ --max-num-batched-tokens "$((ISL * 2 ))" \ --stream-interval 20 --no-enable-prefix-caching \ +--enable-flashinfer-autotune \ --trust-remote-code > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/benchmarks/single_node/fixed_seq_len/patches/flashinfer-pr-3912.patch b/benchmarks/single_node/fixed_seq_len/patches/flashinfer-pr-3912.patch new file mode 100644 index 0000000000..f9759f96f5 --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/patches/flashinfer-pr-3912.patch @@ -0,0 +1,67 @@ +diff --git a/flashinfer/fused_moe/core.py b/flashinfer/fused_moe/core.py +--- a/flashinfer/fused_moe/core.py ++++ b/flashinfer/fused_moe/core.py +@@ -86,6 +86,31 @@ from .utils import ( + ) + + ++@functools.cache ++def _moe_topk_ids_init(num_experts: int): ++ """Return a packed-topk-ids initializer for a given expert count. Cached for ++ object identity preservation. ++ """ ++ ++ def _init( ++ shapes: tuple[int, ...], ++ dtype: torch.dtype, ++ device: torch.device, ++ ) -> torch.Tensor: ++ expert_ids = make_random_topk_ids( ++ num_experts=num_experts, ++ num_tokens=math.prod(shapes[:-1]), ++ top_k=shapes[-1], ++ device=device, ++ ).view(shapes) ++ expert_weights = torch.ones(shapes, dtype=torch.bfloat16, device=device).view( ++ torch.int16 ++ ) ++ return (expert_ids << 16) | expert_weights ++ ++ return _init ++ ++ + # Routing input modes for FusedMoE launcher + # Please keep this in sync with the counterpart defined in csrc/trtllm_fused_moe_kernel_launcher.cu + class RoutingInputMode(IntEnum): +@@ -1250,30 +1275,14 @@ class FusedMoE: + **kwargs: Extra TuningConfig kwargs (e.g. use_cold_l2_cache). + """ + +- def _init_packed_topk_ids( +- shapes: tuple[int, ...], +- dtype: torch.dtype, +- device: torch.device, +- ) -> torch.Tensor: +- expert_ids = make_random_topk_ids( +- num_experts=self.num_experts, +- num_tokens=math.prod(shapes[:-1]), +- top_k=shapes[-1], +- device=device, +- ).view(shapes) +- expert_weights = torch.ones( +- shapes, dtype=torch.bfloat16, device=device +- ).view(torch.int16) +- return (expert_ids << 16) | expert_weights +- + spec = { + "output": autotuner_initializer_empty, + "hidden_states": autotuner_initializer_randn, + } + if moe_inputs.routing_logits is not None: + spec["routing_logits"] = autotuner_initializer_rand + if moe_inputs.topk_ids is not None: +- spec["topk_ids"] = _init_packed_topk_ids ++ spec["topk_ids"] = _moe_topk_ids_init(self.num_experts) + if moe_inputs.expert_weights is not None: + spec["expert_weights"] = autotuner_initializer_ones + if moe_inputs.hidden_states_scale is not None: diff --git a/benchmarks/single_node/fixed_seq_len/patches/vllm-flashinfer-autotune-process-group.patch b/benchmarks/single_node/fixed_seq_len/patches/vllm-flashinfer-autotune-process-group.patch new file mode 100644 index 0000000000..2d34f89f77 --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/patches/vllm-flashinfer-autotune-process-group.patch @@ -0,0 +1,27 @@ +diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py +--- a/vllm/model_executor/warmup/kernel_warmup.py ++++ b/vllm/model_executor/warmup/kernel_warmup.py +@@ -178,9 +178,16 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: + if not use_persistent_cache: +- with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs): +- runner._dummy_run( +- num_tokens=runner.scheduler_config.max_num_batched_tokens, +- skip_eplb=True, +- is_profile=True, +- ) +- get_world_group().barrier() ++ from flashinfer.autotuner import set_autotune_process_group ++ ++ world = get_world_group() ++ set_autotune_process_group(world.cpu_group) ++ try: ++ with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs): ++ runner._dummy_run( ++ num_tokens=runner.scheduler_config.max_num_batched_tokens, ++ skip_eplb=True, ++ is_profile=True, ++ ) ++ finally: ++ set_autotune_process_group(None) ++ world.barrier() + return diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 73327351e1..9661ce0d36 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -13697,7 +13697,7 @@ minimaxm3-fp8-b300-vllm: # weights are pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added to # the STAGED_MODELS allow-list in launch_b300-nv.sh). minimaxm3-fp4-b300-vllm: - image: vllm/vllm-openai:nightly-93d8f834dd8acf33eb0e2a75b2711b628cb6e226 + image: vllm/vllm-openai:nightly-3d99b0499aff7ce3b91942b0385558dab4eecf76 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: b300 @@ -13706,22 +13706,13 @@ minimaxm3-fp4-b300-vllm: multinode: false scenarios: fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 2 } - - { tp: 4, conc-start: 1, conc-end: 64 } - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 4096, conc-end: 4096 } - isl: 8192 osl: 1024 search-space: + - { tp: 1, conc-start: 1, conc-end: 16 } - { tp: 8, conc-start: 1, conc-end: 2 } - { tp: 4, conc-start: 1, conc-end: 2 } - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 } # EAGLE3 speculative-decoding (spec-decoding: mtp) variant of MiniMax-M3 NVFP4 # (nvidia/MiniMax-M3-NVFP4) B300 single-node vLLM, pairing the target with the diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d77c35f1fd..45234d2291 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4781,3 +4781,38 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - minimaxm3-fp4-b300-vllm + description: + - "Update the vLLM image to nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" + - "Install FlashInfer 0.6.15.dev20260712" + - "Use the upstream SM100 low-M MXFP8 split-K implementation included in FlashInfer 0.6.15" + - "Use the upstream FlashInfer PR #3187 distributed AutoTuner synchronization API and PR #3918 non-Tensor guard included in the 0712 nightly" + - "Apply the FlashInfer PR #3912 AutoTuner cache and memory fix via a local runtime patch" + - "Keep the upstream FlashInfer PR #3582, PR #3687, and PR #3745 runtime changes" + - "Patch vLLM to synchronize distributed FlashInfer tactic selection over its gloo world group and re-enable AutoTuner" + - "Set VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=300 to fail GPU hangs faster" + - "Use the explicit FlashInfer TRT-LLM all-reduce backend override" + - "Remove MiniMax 1k1k and 8k1k DEP2 points from this performance bisect" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2124 + +- config-keys: + - minimaxm3-fp4-b300-vllm + description: + - "Update the vLLM image to nightly-3d99b0499aff7ce3b91942b0385558dab4eecf76, the earliest nightly containing vLLM PR #48268" + - "Skip FlashInfer AutoTuner profiling for flashinfer::trtllm_fp4_block_scale_moe while retaining synchronized tuning for the remaining operations" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2124 + +- config-keys: + - minimaxm3-fp4-b300-vllm + description: + - "Add an 8k1k TP1 diagnostic sweep at concurrency 1-16" + - "Use vLLM GPU memory utilization 0.97 for TP1 while retaining 0.95 for all other TP sizes" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2124 + +- config-keys: + - minimaxm3-fp4-b300-vllm + description: + - "Set --max-num-seqs 16 and FULL_DECODE_ONLY CUDA Graph mode for TP1 only" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2124