perf: add GB200 Qwen3.5 FP4 SGLang MTP AgentX

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp.yaml

@@ -0,0 +1,78 @@
+name: qwen35-gb200-sglang-agentic-mtp-agg-tp2ep2
+
+model: { path: qwen3.5-fp4, container: dynamo-sglang, precision: fp4 }
+identity:
+  model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 }
+  container: { image: lmsysorg/sglang:v0.5.17-cu130 }
+slurm: { time_limit: "8:00:00" }
+health_check: { max_attempts: 2160, interval_seconds: 10 }
+resources: { gpu_type: gb200, gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 2 }
+infra: { nats_max_payload_mb: 8 }
+frontend:
+  type: sglang
+  args:
+    worker-startup-timeout-secs: 3600
+
+backend:
+  type: sglang
+  aggregated_environment:
+    PYTHONNOUSERSITE: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    SGLANG_ENABLE_FLASHINFER_GEMM: "true"
+    SGLANG_ENABLE_SPEC_V2: "1"
+    SGL_ENABLE_JIT_DEEPGEMM: "false"
+    TORCH_CUDA_ARCH_LIST: "10.0"
+  sglang_config:
+    aggregated:
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 2
+      data-parallel-size: 1
+      expert-parallel-size: 2
+      enable-symm-mem: true
+      quantization: modelopt_fp4
+      fp4-gemm-backend: flashinfer_cutlass
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      mamba-scheduler-strategy: extra_buffer
+      mamba-track-interval: 8192
+      attention-backend: trtllm_mha
+      linear-attn-decode-backend: flashinfer
+      moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs: 64
+      max-running-requests: 40
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.85
+      max-mamba-cache-size: 200
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      mamba-max-states-per-path: 1
+      enable-metrics: true
+      enable-cache-report: true
+
+sbatch_directives: { mem: "0", cpus-per-task: "144" }
+srun_options: { mem: "0", container-remap-root: "" }
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "false"
+    TP: "2"
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:"

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-hicache.yaml

@@ -0,0 +1,85 @@
+name: qwen35-gb200-sglang-agentic-mtp-agg-tp4-hicache
+
+model: { path: qwen3.5-fp4, container: dynamo-sglang, precision: fp4 }
+identity:
+  model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 }
+  container: { image: lmsysorg/sglang:v0.5.17-cu130 }
+slurm: { time_limit: "8:00:00" }
+health_check: { max_attempts: 2160, interval_seconds: 10 }
+resources: { gpu_type: gb200, gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 }
+infra: { nats_max_payload_mb: 8 }
+frontend:
+  type: sglang
+  args:
+    worker-startup-timeout-secs: 3600
+
+backend:
+  type: sglang
+  aggregated_environment:
+    PYTHONNOUSERSITE: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    SGLANG_ENABLE_FLASHINFER_GEMM: "true"
+    SGLANG_ENABLE_SPEC_V2: "1"
+    SGL_ENABLE_JIT_DEEPGEMM: "false"
+    TORCH_CUDA_ARCH_LIST: "10.0"
+  sglang_config:
+    aggregated:
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 4
+      data-parallel-size: 1
+      expert-parallel-size: 1
+      enable-symm-mem: true
+      quantization: modelopt_fp4
+      fp4-gemm-backend: flashinfer_cutlass
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      mamba-scheduler-strategy: extra_buffer
+      mamba-track-interval: 8192
+      attention-backend: trtllm_mha
+      linear-attn-decode-backend: flashinfer
+      moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs: 64
+      max-running-requests: 160
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.78
+      max-mamba-cache-size: 360
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      tokenizer-worker-num: 6
+      page-size: 64
+      enable-hierarchical-cache: true
+      hicache-ratio: 0.70
+      hicache-io-backend: kernel
+      hicache-mem-layout: page_first_direct
+      hicache-write-policy: write_back
+      mamba-max-states-per-path: 1
+      enable-metrics: true
+      enable-cache-report: true
+
+sbatch_directives: { mem: "0", cpus-per-task: "144" }
+srun_options: { mem: "0", container-remap-root: "" }
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "false"
+    TP: "4"
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:"

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml

@@ -0,0 +1,92 @@
+name: qwen35-gb200-sglang-agentic-mtp-agg-tp4
+
+model:
+  path: qwen3.5-fp4
+  container: dynamo-sglang
+  precision: fp4
+
+identity:
+  model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 }
+  container: { image: lmsysorg/sglang:v0.5.17-cu130 }
+
+slurm: { time_limit: "8:00:00" }
+health_check: { max_attempts: 2160, interval_seconds: 10 }
+
+resources:
+  gpu_type: gb200
+  gpus_per_node: 4
+  agg_nodes: 1
+  agg_workers: 1
+  gpus_per_agg: 4
+
+infra: { nats_max_payload_mb: 8 }
+frontend:
+  type: sglang
+  args:
+    worker-startup-timeout-secs: 3600
+
+backend:
+  type: sglang
+  aggregated_environment:
+    PYTHONNOUSERSITE: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    SGLANG_ENABLE_FLASHINFER_GEMM: "true"
+    SGLANG_ENABLE_SPEC_V2: "1"
+    SGL_ENABLE_JIT_DEEPGEMM: "false"
+    TORCH_CUDA_ARCH_LIST: "10.0"
+  sglang_config:
+    aggregated:
+      served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
+      model-path: /model/
+      trust-remote-code: true
+      tensor-parallel-size: 4
+      data-parallel-size: 1
+      expert-parallel-size: 1
+      enable-symm-mem: true
+      quantization: modelopt_fp4
+      fp4-gemm-backend: flashinfer_cutlass
+      kv-cache-dtype: fp8_e4m3
+      mamba-ssm-dtype: bfloat16
+      mamba-scheduler-strategy: extra_buffer
+      mamba-track-interval: 8192
+      attention-backend: trtllm_mha
+      linear-attn-decode-backend: flashinfer
+      moe-runner-backend: flashinfer_trtllm
+      speculative-algorithm: NEXTN
+      speculative-num-steps: 3
+      speculative-eagle-topk: 1
+      speculative-num-draft-tokens: 4
+      cuda-graph-max-bs: 64
+      max-running-requests: 160
+      max-prefill-tokens: 16384
+      chunked-prefill-size: 16384
+      mem-fraction-static: 0.80
+      max-mamba-cache-size: 360
+      allow-auto-truncate: true
+      stream-interval: 50
+      scheduler-recv-interval: 10
+      tokenizer-worker-num: 6
+      mamba-max-states-per-path: 1
+      enable-metrics: true
+      enable-cache-report: true
+
+sbatch_directives: { mem: "0", cpus-per-task: "144" }
+srun_options: { mem: "0", container-remap-root: "" }
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "false"
+    TP: "4"
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:"

configs/nvidia-master.yaml

@@ -7194,6 +7194,64 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32] }
 
+qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp:
+  image: lmsysorg/sglang:v0.5.17-cu130
+  model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
+  model-prefix: qwen3.5
+  runner: cluster:gb200-nv
+  precision: fp4
+  framework: dynamo-sglang
+  multinode: true
+  disagg: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.70
+      search-space:
+      # Measured Pareto points from the complete fast sweep. TP4 covers the
+      # low-latency branch; HiCache extends the high-throughput branch.
+      - spec-decoding: mtp
+        kv-offloading: none
+        conc-list: [1, 4, 8, 12, 16, 20, 32, 40]
+        prefill:
+          num-worker: 1
+          tp: 4
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "SYNTHETIC_ACCEPTANCE=true"
+          - "SYNTHETIC_ACCEPTANCE_LENGTH=3.39"
+          - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml"
+        decode: { num-worker: 0, tp: 4, ep: 1, dp-attn: false }
+      - spec-decoding: mtp
+        kv-offloading: dram
+        kv-offload-backend: { name: hicache }
+        conc-list: [48, 56, 64, 72]
+        prefill:
+          num-worker: 1
+          tp: 4
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "SYNTHETIC_ACCEPTANCE=true"
+          - "SYNTHETIC_ACCEPTANCE_LENGTH=3.39"
+          - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-hicache.yaml"
+        decode: { num-worker: 0, tp: 4, ep: 1, dp-attn: false }
+      # TP2/EP2 supplies the middle of the measured frontier with half the
+      # GPU count of TP4.
+      - spec-decoding: mtp
+        kv-offloading: none
+        conc-list: [4, 8, 16, 20, 24]
+        prefill:
+          num-worker: 1
+          tp: 2
+          ep: 2
+          dp-attn: false
+          additional-settings:
+          - "SYNTHETIC_ACCEPTANCE=true"
+          - "SYNTHETIC_ACCEPTANCE_LENGTH=3.39"
+          - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp.yaml"
+        decode: { num-worker: 0, tp: 2, ep: 2, dp-attn: false }
+
 minimaxm3-fp4-b300-vllm-agentic-mtp:
   image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7
   model: nvidia/MiniMax-M3-NVFP4

perf-changelog.yaml

@@ -5845,3 +5845,14 @@
   description:
     - "Add MiniMax-M3 MXFP8 AgentX on H100 with vLLM v0.27.1, EAGLE3 golden AL 2.78, resident TP8 c1-c5, and Mooncake DRAM offload c6/c8."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2564
+
+- config-keys:
+    - qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp
+  description:
+    - "Add the measured GB200 Qwen3.5 NVFP4 AgentX Pareto frontier across aggregate TP4, TP2/EP2, and TP4 HiCache with K=3 MTP."
+    - "Use stable SGLang v0.5.17-cu130 and NVIDIA/srt-slurm v1.0.45."
+    - "Enable SGLang metrics/cache reporting and require nonempty sglang: server metrics before publishing results."
+    - "Use K=3 golden synthetic acceptance length 3.39 for throughput; eval retains real MTP verification."
+    - "Give the released SGLang router one hour to register the 28-minute-loading backend and route AgentX sessions by correlation ID."
+    - "Bound hybrid-state pools to the measured resident and HiCache operating ranges instead of reserving capacity that cannot fit beside target and MTP weights."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2568

runners/launch_gb200-nv.sh

@@ -178,6 +178,9 @@ if [[ $FRAMEWORK == "dynamo-sglang" ]]; then
     elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then
         export MODEL_PATH="/mnt/lustre01/models/Qwen3.5-397B-A17B-FP8"
         export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8"
+    elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp4" ]]; then
+        export MODEL_PATH="/mnt/lustre01/models/Qwen3.5-397B-A17B-NVFP4-V2"
+        export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp4"
     elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then
         # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5-fp4")
         # in our GLM-5.1 sglang recipes.
@@ -388,8 +391,20 @@ if [ -d "$SRT_REPO_DIR" ]; then
     rm -rf "$SRT_REPO_DIR"
 fi
 
-# TODO(CJQ): make first class upon srt-slurm upstream refactor
-if [[ "$IS_AGENTIC" == "1" ]]; then
+# Qwen3.5 FP4 AgentX uses the latest released srt-slurm. v1.0.45 injects
+# the aggregate logical-worker Prometheus endpoint into custom benchmarks.
+if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then
+    git clone --branch v1.0.45 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
+    cd "$SRT_REPO_DIR"
+    test "$(git rev-parse HEAD)" = "9d8d92b20c350a5d42f0709f5a0b64e30eb37d33" || {
+        echo "Error: NVIDIA/srt-slurm v1.0.45 resolved to an unexpected commit" >&2
+        exit 1
+    }
+    mkdir -p recipes/sglang/qwen3.5/gb200-fp4/agentic
+    cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic" \
+        recipes/sglang/qwen3.5/gb200-fp4/agentic
+# TODO(CJQ): migrate the remaining Agentic model paths to released srt-slurm.
+elif [[ "$IS_AGENTIC" == "1" ]]; then
     # Agentic multi-node pins cquil11/srt-slurm-nv revisions that provide:
     #   - BenchmarkType.CUSTOM + benchmark.command + benchmark.env
     #     (the hook that hands off to benchmarks/multi_node/agentic_srt.sh)

runners/synthetic_injectors/__init__.py

@@ -35,4 +35,4 @@ def get_injector(framework):
 
 # Import backends after register/get_injector are defined so each module can
 # call register() at import time. Add new frameworks (sglang, trtllm, ...) here.
-from . import vllm  # noqa: E402,F401
+from . import sglang, vllm  # noqa: E402,F401