[AgentX] Test vLLM DeepSeek-V4 GB300 aggregate TP / 测试 vLLM DeepSeek-V4 GB300 聚合式 TP

benchmarks/benchmark_lib.sh

@@ -1797,7 +1797,11 @@ build_replay_cmd() {
     # X-Correlation-ID is useful tracing metadata but does not establish that
     # binding by itself. AIPerf emits nvext.session_control bind/close actions
     # keyed by the stable conversation correlation ID when this flag is set.
-    if [[ "${FRAMEWORK:-}" == dynamo-* ]]; then
+    # Opt-out: recipes set AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0 to skip this.
+    # aiperf's conv-aware routing emits nvext.session_control, a removed POC field
+    # (dynamo #9920 / v1.3.0-dev) that current dynamo builds reject with a 400
+    # (they moved to router/routing_constraints/agent_context). Default stays on.
+    if [[ "${FRAMEWORK:-}" == dynamo-* && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" ]]; then
         REPLAY_CMD+=" --use-dynamo-conv-aware-routing"
         # The upstream 300s affinity TTL is shorter than an overloaded
         # high-concurrency agentic request. Keep bindings alive across long

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml

@@ -0,0 +1,147 @@
+name: "svf-vllm-agg-gb300-tp4-agentic"
+
+# Low-latency aggregate DeepSeek-V4-Pro AgentX recipe on one four-GPU GB300
+# node. The worker serves both prefill and decode with TP4.
+
+model:
+  path: "deepseek-v4-pro"
+  container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "deepseek-ai/DeepSeek-V4-Pro"
+  container:
+    image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96"
+  frameworks:
+    dynamo: "1.2.1"
+    vllm: "0.17.2rc1.dev3675+gc188b96eb"
+
+dynamo:
+  hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0"
+  install: true
+
+setup_script: vllm-container-deps.sh
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 2160
+  interval_seconds: 10
+
+resources:
+  gpu_type: "gb300"
+  gpus_per_node: 4
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 4
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  args:
+    router-mode: "kv"
+    router-reset-states: true
+    router-temperature: 0.0
+    router-queue-threshold: 65536
+    active-decode-blocks-threshold: "None"
+    active-prefill-tokens-threshold: "None"
+    active-prefill-tokens-threshold-frac: "None"
+    tokenizer: "fastokens"
+
+backend:
+  type: vllm
+  connector: null
+  mooncake_kv_store:
+    store_config:
+      metadata_server: "P2PHANDSHAKE"
+      global_segment_size: "150GB"
+      local_buffer_size: "4GB"
+      protocol: "rdma"
+      device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+      mode: "embedded"
+      enable_offload: false
+  aggregated_environment:
+    DYN_REQUEST_PLANE: "tcp"
+    ETCD_LEASE_TTL: "120"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    VLLM_ALLREDUCE_USE_SYMM_MEM: "0"
+    VLLM_USE_BREAKABLE_CUDAGRAPH: "0"
+    TORCH_SYMMMEM: "NVSHMEM"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_MOONCAKE_LOAD_RECV_THREADS: "4"
+    VLLM_CONNECTOR_PREFETCH_DEPTH: "8"
+    VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65"
+    DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}"
+    VLLM_DSV4_MEGA_FP8_COMBINE: "1"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_RCACHE_MAX_UNRELEASED: "1024"
+    UCX_TLS: "cuda_copy,cuda_ipc,tcp"
+    NCCL_P2P_LEVEL: NVL
+    MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
+
+  vllm_config:
+    aggregated:
+      kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}'
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      kv-cache-dtype: "fp8"
+      tensor-parallel-size: 4
+      pipeline-parallel-size: 1
+      enable-cumem-allocator: true
+      numa-bind: true
+      numa-bind-nodes: [0, 0, 1, 1]
+      attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
+      max-model-len: 1048576
+      max-num-seqs: 32
+      max-num-batched-tokens: 16384
+      trust-remote-code: true
+      no-enable-flashinfer-autotune: true
+      block-size: 256
+      compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
+      max-cudagraph-capture-size: 32
+      gpu-memory-utilization: 0.92
+      stream-interval: 10
+      no-disable-hybrid-kv-cache-manager: true
+      tokenizer-mode: deepseek_v4
+      dyn-tool-call-parser: deepseek_v4
+      reasoning-parser: deepseek_v4
+      dyn-reasoning-parser: deepseek_v4
+
+sbatch_directives:
+  cpus-per-task: "72"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "false"
+    TP: "4"
+    EP_SIZE: "1"
+    DP_ATTENTION: "false"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml

@@ -0,0 +1,147 @@
+name: "svf-vllm-agg-gb300-tp8-agentic"
+
+# Low-latency aggregate DeepSeek-V4-Pro AgentX recipe spanning two four-GPU
+# GB300 nodes. The worker serves both prefill and decode with TP8.
+
+model:
+  path: "deepseek-v4-pro"
+  container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "deepseek-ai/DeepSeek-V4-Pro"
+  container:
+    image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96"
+  frameworks:
+    dynamo: "1.2.1"
+    vllm: "0.17.2rc1.dev3675+gc188b96eb"
+
+dynamo:
+  hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0"
+  install: true
+
+setup_script: vllm-container-deps.sh
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 2160
+  interval_seconds: 10
+
+resources:
+  gpu_type: "gb300"
+  gpus_per_node: 4
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 8
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  args:
+    router-mode: "kv"
+    router-reset-states: true
+    router-temperature: 0.0
+    router-queue-threshold: 65536
+    active-decode-blocks-threshold: "None"
+    active-prefill-tokens-threshold: "None"
+    active-prefill-tokens-threshold-frac: "None"
+    tokenizer: "fastokens"
+
+backend:
+  type: vllm
+  connector: null
+  mooncake_kv_store:
+    store_config:
+      metadata_server: "P2PHANDSHAKE"
+      global_segment_size: "150GB"
+      local_buffer_size: "4GB"
+      protocol: "rdma"
+      device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+      mode: "embedded"
+      enable_offload: false
+  aggregated_environment:
+    DYN_REQUEST_PLANE: "tcp"
+    ETCD_LEASE_TTL: "120"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    VLLM_ALLREDUCE_USE_SYMM_MEM: "0"
+    VLLM_USE_BREAKABLE_CUDAGRAPH: "0"
+    TORCH_SYMMMEM: "NVSHMEM"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    VLLM_USE_RUST_FRONTEND: "1"
+    VLLM_MOONCAKE_LOAD_RECV_THREADS: "4"
+    VLLM_CONNECTOR_PREFETCH_DEPTH: "8"
+    VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65"
+    DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}"
+    VLLM_DSV4_MEGA_FP8_COMBINE: "1"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_RCACHE_MAX_UNRELEASED: "1024"
+    UCX_TLS: "cuda_copy,cuda_ipc,tcp"
+    NCCL_P2P_LEVEL: NVL
+    MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
+
+  vllm_config:
+    aggregated:
+      kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}'
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      kv-cache-dtype: "fp8"
+      tensor-parallel-size: 8
+      pipeline-parallel-size: 1
+      enable-cumem-allocator: true
+      numa-bind: true
+      numa-bind-nodes: [0, 0, 1, 1]
+      attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
+      max-model-len: 1048576
+      max-num-seqs: 32
+      max-num-batched-tokens: 32768
+      trust-remote-code: true
+      no-enable-flashinfer-autotune: true
+      block-size: 256
+      compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
+      max-cudagraph-capture-size: 32
+      gpu-memory-utilization: 0.92
+      stream-interval: 10
+      no-disable-hybrid-kv-cache-manager: true
+      tokenizer-mode: deepseek_v4
+      dyn-tool-call-parser: deepseek_v4
+      reasoning-parser: deepseek_v4
+      dyn-reasoning-parser: deepseek_v4
+
+sbatch_directives:
+  cpus-per-task: "72"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "false"
+    TP: "8"
+    EP_SIZE: "1"
+    DP_ATTENTION: "false"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

configs/nvidia-master.yaml

@@ -7663,6 +7663,48 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8:
           ep: 8
           dp-attn: true
 
+dsv4-fp4-gb300-dynamo-vllm-agentic-agg:
+  image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4
+  runner: cluster:gb300-nv
+  precision: fp4
+  framework: dynamo-vllm
+  router: { name: dynamo-router, version: "1.2.1" }
+  multinode: true
+  disagg: false
+  scenarios:
+    agentic-coding:
+    - search-space:
+      - spec-decoding: none
+        conc-list: [4]
+        prefill:
+          num-worker: 1
+          tp: 4
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 4
+          ep: 1
+          dp-attn: false
+      - spec-decoding: none
+        conc-list: [1]
+        prefill:
+          num-worker: 1
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 8
+          ep: 1
+          dp-attn: false
+
 dsv4-fp4-gb300-dynamo-sglang-agentic-agg:
   image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8
   model: deepseek-ai/DeepSeek-V4-Pro

perf-changelog.yaml

@@ -4974,3 +4974,11 @@
     - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity"
     - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-vllm-agentic-agg
+  description:
+    - "Test only the GB300 Dynamo-vLLM AgentX aggregate TP4 at conc 4 and TP8 at conc 1."
+    - "Set gpu-memory-utilization to 0.92 for TP4/TP8 and enable the DeepSeek-V4 tool-call and reasoning parsers."
+    - "Use vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 for all new recipes."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2284