[NVIDIA][AgentX] GB300 DeepSeek-V4 dynamo-sglang agentic recipes

.github/workflows/benchmark-multinode-tmpl.yml

@@ -240,7 +240,7 @@ jobs:
       ${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }}
       ${{ inputs.kv-offloading != '' && inputs.kv-offloading != 'none' && format('{0} KV offload', inputs.kv-offloading) || '' }}
       ${{ inputs.kv-offload-backend != '' && inputs.kv-offload-backend != 'none' && inputs.kv-offload-backend != 'default' && inputs.kv-offload-backend || '' }}
-      c${{ inputs.conc != '' && inputs.conc || join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }}
+      c${{ join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }}
 
     steps:
       - name: Slurm cleanup (pre-run)

.github/workflows/run-sweep.yml

@@ -556,7 +556,7 @@ jobs:
             precision: ${{ matrix.config.precision }}
             router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }}
             kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }}
-            conc-list: '[${{ matrix.config.conc }}]'
+            conc-list: ${{ toJson(matrix.config.conc) }}
             spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ matrix.config.disagg }}
             prefill-hardware: ${{ matrix.config.prefill.hardware }}
@@ -577,7 +577,7 @@ jobs:
             decode-ep: ${{ matrix.config.decode.ep }}
             decode-dp-attn: ${{ matrix.config.decode.dp-attn }}
             decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }}
-            conc: ${{ matrix.config.conc }}
+            conc: ${{ matrix.config.conc[0] }}
             kv-offloading: ${{ matrix.config.kv-offloading }}
             kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }}
             kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }}

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml

@@ -0,0 +1,154 @@
+name: "agg-gb300-tp4-mtp-kvoffload"
+
+# Agentic-coding SGLang aggregated recipe for DeepSeek-V4-Pro on GB300
+# (single aggregated worker, TP4, MTP + hierarchical-cache KV offload).
+#
+# Uses the flat single-variant schema the agentic CI flow expects. Concurrency is NOT
+# a recipe field here: the GHA matrix fans out one job per concurrency from
+# the master-config conc-list, exporting CONC into agentic_srt.sh. Modeled on
+# the flat vllm/deepseek-v4/agentic recipes (agentic infra + benchmark wiring)
+# and the flat sglang/deepseek-v4/8k1k recipes (sglang backend schema).
+
+model:
+  path: "deepseek-v4-pro"
+  container: "dynamo-sglang"
+  precision: "fp4"
+
+dynamo:
+  install: true
+  wheel: "1.3.0.dev1"
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 1440
+  interval_seconds: 10
+
+resources:
+  gpu_type: "gb300"
+  gpus_per_node: 4
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 4
+
+infra:
+  # Dedicated etcd/nats node — matches the vllm agentic recipes; under the
+  # dynamo router + multiple frontends the infra services need their own node.
+  etcd_nats_dedicated_node: true
+  # cc-traces prompts (~125K tokens) serialize to ~2.8MB; the 1 MiB NATS
+  # default drops them. 32 MiB matches the agentic vllm recipes (~10x headroom
+  # over the largest observed payload); schema recommends 24+ for long ISL.
+  nats_max_payload_mb: 32
+
+frontend:
+  type: dynamo
+  nginx_session_affinity: true
+  nginx_session_affinity_header: X-Correlation-ID
+  enable_multiple_frontends: true
+  num_additional_frontends: 4
+  env:
+    DYN_ROUTER_TEMPERATURE: "10000000"
+    # The sglang image is PEP 668 externally-managed; the runtime dynamo
+    # install (dynamo_wheels.py / source build) runs pip and fails with
+    # "externally-managed-environment" without this. Lets pip install into
+    # the system env. Applies to both dynamo.hash (source) and dynamo.wheel.
+    PIP_BREAK_SYSTEM_PACKAGES: "1"
+  args:
+    router-mode: "kv"
+    router-reset-states: true
+    active-decode-blocks-threshold: "None"
+    active-prefill-tokens-threshold: "None"
+    active-prefill-tokens-threshold-frac: "None"
+
+backend:
+  type: sglang
+
+  aggregated_environment:
+    SGLANG_DEFAULT_THINKING: '1'
+    SGLANG_DSV4_REASONING_EFFORT: high
+    SGLANG_SIMULATE_ACC_LEN: '2.49'
+    SGLANG_SIMULATE_ACC_METHOD: match-expected
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"
+    # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image.
+    PIP_BREAK_SYSTEM_PACKAGES: "1"
+    # from submission for B300
+    SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1"
+    SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
+    SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1"
+    SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1"
+    SGLANG_OPT_USE_ONLINE_COMPRESS: "0"
+    SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
+    SGLANG_OPT_USE_JIT_NORM: "1"
+    SGLANG_OPT_USE_TOPK_V2: "True"
+    # for kvcache offload
+    SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1"
+
+  sglang_config:
+    aggregated:
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      trust-remote-code: true
+      stream-interval: 50
+      watchdog-timeout: 1000000
+      mem-fraction-static: 0.90
+      chunked-prefill-size: 8192  # can only support 11000 for TP
+      moe-runner-backend: "flashinfer_mxfp4"
+      disable-flashinfer-autotune: true
+      swa-full-tokens-ratio: 0.1
+      max-running-requests: 128
+      cuda-graph-max-bs: 128
+
+      scheduler-recv-interval: 30
+      dp-size: 1
+      tp-size: 4
+      ep-size: 1
+      # kv-cache offload
+      enable-hierarchical-cache: true
+      hicache-ratio: 6
+      hicache-write-policy: write_through
+      hicache-io-backend: kernel
+      # mtp section
+      speculative-algorithm: EAGLE
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+
+sbatch_directives:
+  mem: "0"
+  cpus-per-task: "144"
+
+srun_options:
+  mem: "0"
+  # gb300-nv: pyxis maps the calling user into the container as a non-root
+  # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to
+  # uid 0 inside the container. srt-slurm renders empty-string values as
+  # flag-only srun args.
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    # Aggregated: one worker serves both prefill and decode, so GPU accounting
+    # is the single-worker form num_gpus = TP (not the disagg prefill+decode
+    # sum). The multinode post-processing path assumes disagg and would divide
+    # throughput by prefill_gpus + decode_gpus; force the single-node code path
+    # in process_agentic_result.py instead. TP must match sglang_config tp-size.
+    IS_MULTINODE: "false"
+    TP: "4"
+    # Enable dynamo conv-aware routing (per-session affinity so multi-turn
+    # KV cache is reused across turns). Previously opted out because the
+    # frontend 400'd on aiperf's nvext.session_control actions; re-enabled
+    # to test with the current build.
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1"
+    # Container-side path of the aiperf mmap dataset cache; host-side mount
+    # wired via launch_gb300-*.sh srtslurm.yaml default_mounts. Without it,
+    # aiperf re-tokenizes + re-writes the dataset mmap on every run.
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    # Persistent HF hub cache (also via default_mounts) so the trace dataset
+    # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE.
+    HF_HUB_CACHE: "/hf_hub_cache"
+

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml

@@ -0,0 +1,212 @@
+name: "disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload"
+
+# Agentic-coding SGLang disaggregated Pareto recipe for DeepSeek-V4-Pro on GB300
+# (12P x DEP8 / 4D x DEP16, MTP + hierarchical-cache KV offload), tuned for concurrency 1024.
+#
+# Uses the flat single-variant srtctl schema the agentic CI flow expects;
+# resources + backend (prefill/decode env + sglang_config) are normalized
+# from the Pareto run.
+# Concurrency is exported into agentic_srt.sh from the master-config conc-list.
+
+model:
+  path: "deepseek-v4-pro"
+  container: "dynamo-sglang"
+  precision: "fp4"
+
+dynamo:
+  install: true
+  wheel: "1.3.0.dev1"
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 1440
+  interval_seconds: 10
+
+resources:
+  gpu_type: gb300
+  gpus_per_node: 4
+  prefill_nodes: 12
+  decode_nodes: 4
+  prefill_workers: 6
+  decode_workers: 1
+  gpus_per_prefill: 8
+  gpus_per_decode: 16
+
+infra:
+  etcd_nats_dedicated_node: true
+  nats_max_payload_mb: 32
+
+frontend:
+  type: dynamo
+  nginx_session_affinity: true
+  nginx_session_affinity_header: X-Correlation-ID
+  enable_multiple_frontends: true
+  num_additional_frontends: 4
+  env:
+    DYN_ROUTER_TEMPERATURE: "10000000"
+    PIP_BREAK_SYSTEM_PACKAGES: "1"
+  args:
+    router-mode: "kv"
+    router-reset-states: true
+    active-decode-blocks-threshold: "None"
+    active-prefill-tokens-threshold: "None"
+    active-prefill-tokens-threshold-frac: "None"
+
+backend:
+  type: sglang
+
+  prefill_environment:
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900"
+    SGLANG_DSV4_MHC_PREWARM: '1'
+    PIP_BREAK_SYSTEM_PACKAGES: '1'
+    PYTHONUNBUFFERED: '1'
+    SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
+    SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache
+    SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1'
+    SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1'
+    SGLANG_DEFAULT_THINKING: '1'
+    SGLANG_DSV4_REASONING_EFFORT: high
+    SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1'
+    SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1"
+    SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1'
+    SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1'
+    SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1'
+    SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1'
+    SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216'
+    SGLANG_OPT_USE_ONLINE_COMPRESS: '0'
+    NCCL_MNNVL_ENABLE: '1'
+    NCCL_CUMEM_ENABLE: '1'
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
+    MC_FORCE_MNNVL: '1'
+    NCCL_TIMEOUT: '100000'
+    NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
+    SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
+    DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
+    DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn
+    SGLANG_LOG_FORWARD_ITERS: '1'
+    SGLANG_LOG_MS: '1'
+    SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
+    SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1'
+
+  decode_environment:
+    SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900"
+    SGLANG_SIMULATE_ACC_LEN: '2.49'
+    SGLANG_SIMULATE_ACC_METHOD: match-expected
+    SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"
+    PIP_BREAK_SYSTEM_PACKAGES: '1'
+    PYTHONUNBUFFERED: '1'
+    SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
+    SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache
+    SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1'
+    SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1'
+    SGLANG_DEFAULT_THINKING: '1'
+    SGLANG_DSV4_REASONING_EFFORT: high
+    SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1'
+    SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1"
+    SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1'
+    SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: '1'
+    SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: '1'
+    SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096'
+    SGLANG_OPT_USE_ONLINE_COMPRESS: '0'
+    NCCL_MNNVL_ENABLE: '1'
+    NCCL_CUMEM_ENABLE: '1'
+    SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
+    MC_FORCE_MNNVL: '1'
+    NCCL_TIMEOUT: '100000'
+    NCCL_MNNVL_UUID_QUERY_TIMEOUT: '100000'
+    SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1'
+    SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8'
+    DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
+    DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn
+    SGLANG_LOG_FORWARD_ITERS: '1'
+    SGLANG_LOG_MS: '1'
+    SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60'
+    SGLANG_ENABLE_UNIFIED_RADIX_TREE: '1'
+
+  sglang_config:
+    prefill:
+      host: 0.0.0.0
+      served-model-name: deepseek-ai/DeepSeek-V4-Pro
+      model-path: /model/
+      trust-remote-code: true
+      watchdog-timeout: 86400
+      stream-interval: 60
+      tp-size: 8
+      dp-size: 8
+      ep-size: 8
+      enable-dp-attention: true
+      enable-dp-lm-head: true
+      moe-dense-tp-size: 1
+      moe-a2a-backend: megamoe
+      disaggregation-transfer-backend: mooncake
+      disaggregation-mode: prefill
+      load-balance-method: total_tokens
+      mem-fraction-static: 0.9
+      swa-full-tokens-ratio: 0.02
+      max-running-requests: 1024
+      cuda-graph-max-bs: 1024
+      chunked-prefill-size: 32768
+      disable-flashinfer-autotune: true
+      model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}'
+      kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}'
+      speculative-algorithm: EAGLE
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      enable-hierarchical-cache: true
+      hicache-write-policy: write_back
+      hicache-ratio: 1
+      hicache-io-backend: direct
+
+    decode:
+      host: 0.0.0.0
+      served-model-name: deepseek-ai/DeepSeek-V4-Pro
+      model-path: /model/
+      trust-remote-code: true
+      watchdog-timeout: 86400
+      stream-interval: 60
+      tp-size: 16
+      dp-size: 16
+      ep-size: 16
+      enable-dp-attention: true
+      enable-dp-lm-head: true
+      moe-dense-tp-size: 1
+      moe-a2a-backend: megamoe
+      disaggregation-transfer-backend: mooncake
+      disaggregation-mode: decode
+      load-balance-method: total_tokens
+      mem-fraction-static: 0.9
+      swa-full-tokens-ratio: 0.02
+      max-running-requests: 3072
+      cuda-graph-max-bs: 1024
+      disable-flashinfer-autotune: true
+      model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}'
+      kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}'
+      speculative-algorithm: EAGLE
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+
+
+sbatch_directives:
+  mem: "0"
+  cpus-per-task: "144"
+
+srun_options:
+  mem: "0"
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+