AMD AgentX MI355X: GPU sanity dedup, HiCache tuning, node-exclude scoping, and DSv4 disagg image bump

benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh

@@ -84,18 +84,11 @@ if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; t
   # DSV4 uses page-size 256 (set in models.yaml); HiCache must match.
   export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}"
   # HiCache ratio (host pool = ratio * GPU KV pool).
-  export HICACHE_RATIO="${HICACHE_RATIO:-4}"
-  # server_sglang.sh prefers an absolute --hicache-size (derived from
-  # TOTAL_CPU_DRAM_GB, the sweep generator's per-node DRAM budget) over
-  # --hicache-ratio whenever TOTAL_CPU_DRAM_GB is set. DSv4 wants the
-  # ratio-based pool instead. Use FORCE_HICACHE_RATIO to opt out of the
-  # --hicache-size path rather than unsetting TOTAL_CPU_DRAM_GB itself:
-  # that var is also the shared client-side gate (benchmark_lib.sh requires
-  # it to be a positive integer whenever KV_OFFLOADING=dram) and gets
-  # forwarded into the aiperf sibling container's client.env, so unsetting
-  # it here made the client fail its own env validation before benchmarking
-  # ("DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB
-  # capacity") even though the servers came up fine.
+  export HICACHE_RATIO="${HICACHE_RATIO:-3}"
+  # DSv4 wants the ratio-based pool, but server_sglang.sh prefers
+  # --hicache-size over --hicache-ratio when TOTAL_CPU_DRAM_GB is set.
+  # Opt out via FORCE_HICACHE_RATIO instead of unsetting TOTAL_CPU_DRAM_GB
+  # (also required client-side by benchmark_lib.sh when KV_OFFLOADING=dram).
   export FORCE_HICACHE_RATIO=1
 
   # ── HiCache layout/backend by tier ──

benchmarks/multi_node/amd_utils/job.slurm

@@ -676,21 +676,21 @@ else
 fi
 fi  # end: if ENGINE == atom-disagg
 
-# Pre-clean (idempotent): stop then force-remove so GPU VRAM is released before
-# the sanity gate. stop-only left containers in Created/Exited state on some nodes.
+# Pre-clean (idempotent): stop then force-remove so GPU VRAM is released
+# before the drain gate. stop-only left containers in Created/Exited state
+# on some nodes.
 \$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true
 \$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop -t 15 || true
 \$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD rm -f || true
 sleep 2
 
-# GPU sanity gate: containers are stopped, so any remaining VRAM use is a bare
-# (non-containerized) process hogging the GPU -- fail fast (and name it)
-# instead of OOMing in model load ~30 min later. set -e + --kill-on-bad-exit
-# tears down the whole job on non-zero exit.
+# GPU drain gate: fail fast on leftover VRAM use instead of OOMing in model
+# load ~15 min later. Reuses wait_for_amd_gpu_clean from benchmark_lib.sh.
 if [[ \"${SKIP_GPU_SANITY:-0}\" == \"1\" ]]; then
-    echo \"[INFO] SKIP_GPU_SANITY=1 set; skipping GPU sanity pre-flight check\"
+    echo \"[INFO] SKIP_GPU_SANITY=1 set; skipping GPU pre-flight drain check\"
 else
-    bash \"$DI_REPO_DIR/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh\"
+    # Unset so benchmark_lib.sh's unrelated agentic KV_OFFLOADING check doesn't exit 1 here.
+    bash -c \"unset IS_AGENTIC SCENARIO_TYPE; source $DI_REPO_DIR/benchmarks/benchmark_lib.sh && wait_for_amd_gpu_clean\"
 fi
 
 # Start vLLM external router container on node 0

benchmarks/multi_node/amd_utils/models.yaml

@@ -369,7 +369,7 @@ DeepSeek-V4-Pro-AgentX:
   ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
   mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1"
   prefill:
-    mem_fraction_static: 0.72
+    mem_fraction_static: 0.85
     disable_radix_cache: false
     disable_cuda_graph: true
     dp:

benchmarks/multi_node/amd_utils/node_excludes.yaml

@@ -0,0 +1,14 @@
+# Per (framework, model) SLURM node exclusions, consumed by submit.sh.
+#
+# A rule applies when FRAMEWORK matches "framework" exactly and MODEL_NAME is
+# in "models"; its exclude_nodes are then passed to sbatch --exclude. Any
+# (framework, model) combination with no matching rule gets an empty exclude
+# list (i.e. no nodes excluded).
+#
+# SLURM_EXCLUDE_NODES env var, if set, always overrides this file.
+
+rules:
+  - framework: vllm-disagg
+    models:
+      - Kimi-K2.5-MXFP4
+    exclude_nodes: mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12

benchmarks/multi_node/amd_utils/submit.sh

@@ -183,12 +183,80 @@ if [[ -n "${NODE_LIST//[[:space:]]/}" ]]; then
     NODELIST_OPT=(--nodelist "$NODELIST_CSV")
 fi
 
-# Optional: exclude specific nodes (e.g. nodes with broken Docker sockets).
-# Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames.
+# Optional: exclude specific nodes for known-bad (FRAMEWORK, MODEL_NAME)
+# combos (e.g. nodes with broken Docker sockets), looked up from
+# node_excludes.yaml. Set SLURM_EXCLUDE_NODES to override with an explicit
+# comma-separated hostname list (takes precedence over the file).
+#
+# Resolution must fail loudly (not silently yield an empty exclude list) if
+# it can't be trusted: a submit host missing python3/PyYAML, or a genuine
+# parse error, must not silently reintroduce the known-bad-node issue this
+# exclusion mechanism exists to prevent.
 EXCLUDE_OPT=()
-SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12}"
+NODE_EXCLUDES_YAML="$(dirname "$0")/node_excludes.yaml"
 if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then
-    EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES")
+    RESOLVED_EXCLUDE_NODES="$SLURM_EXCLUDE_NODES"
+elif [[ -f "$NODE_EXCLUDES_YAML" ]]; then
+    if command -v python3 >/dev/null 2>&1 && python3 -c "import yaml" >/dev/null 2>&1; then
+        RESOLVED_EXCLUDE_NODES=$(python3 -c "
+import yaml
+
+with open('${NODE_EXCLUDES_YAML}') as f:
+    cfg = yaml.safe_load(f) or {}
+
+framework = '${FRAMEWORK}'
+model = '${MODEL_NAME}'
+for rule in cfg.get('rules', []):
+    if rule.get('framework') == framework and model in (rule.get('models') or []):
+        print(rule.get('exclude_nodes', ''))
+        break
+")
+        PYTHON_EXCLUDE_RC=$?
+        if [[ $PYTHON_EXCLUDE_RC -ne 0 ]]; then
+            echo "Error: python3 failed (exit ${PYTHON_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2
+            echo "Error: fix the YAML, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2
+            exit 1
+        fi
+    else
+        # Fall back to an awk parser (mirrors job.slurm's awk-based models.yaml
+        # parsing) matched to node_excludes.yaml's fixed rule/models/exclude_nodes
+        # shape. Only exercised when python3 or its yaml module is unavailable.
+        echo "Warning: python3/PyYAML unavailable on submit host; falling back to awk parsing of ${NODE_EXCLUDES_YAML}" >&2
+        RESOLVED_EXCLUDE_NODES=$(awk -v fw="$FRAMEWORK" -v model="$MODEL_NAME" '
+            /^  - framework:/ {
+                line = $0
+                sub(/^  - framework: */, "", line)
+                fw_match = (line == fw)
+                model_match = 0
+                next
+            }
+            fw_match && /^      - / {
+                m = $0
+                sub(/^      - */, "", m)
+                gsub(/^"|"$/, "", m)
+                if (m == model) model_match = 1
+                next
+            }
+            fw_match && model_match && /^    exclude_nodes:/ {
+                val = $0
+                sub(/^ *exclude_nodes: */, "", val)
+                gsub(/^"|"$/, "", val)
+                print val
+                exit
+            }
+        ' "$NODE_EXCLUDES_YAML")
+        AWK_EXCLUDE_RC=$?
+        if [[ $AWK_EXCLUDE_RC -ne 0 ]]; then
+            echo "Error: awk fallback failed (exit ${AWK_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2
+            echo "Error: fix the YAML/parser, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2
+            exit 1
+        fi
+    fi
+else
+    RESOLVED_EXCLUDE_NODES=""
+fi
+if [[ -n "$RESOLVED_EXCLUDE_NODES" ]]; then
+    EXCLUDE_OPT=(--exclude "$RESOLVED_EXCLUDE_NODES")
 fi
 
 # =============================================================================

configs/amd-master.yaml

@@ -1514,7 +1514,7 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp:
       - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] }
 
 dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp:
-  image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719
+  image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:mi355x-amds
@@ -1528,7 +1528,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp:
     - dram-utilization: 0.80
       search-space:
       - spec-decoding: "mtp"
-        conc-list: [ 1,2,4,8,16,32 ]
+        conc-list: [ 1,2,4,8,16,32,48,64 ]
         kv-offloading: dram
         kv-offload-backend: { name: hicache }
         prefill:
@@ -1548,7 +1548,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp:
           - "DECODE_NODES=1"
           - "DECODE_MTP_SIZE=3"
       - spec-decoding: "mtp"
-        conc-list: [ 96, 128 ]
+        conc-list: [ 64, 96, 128, 192 ]
         kv-offloading: dram
         kv-offload-backend: { name: hicache }
         prefill:

perf-changelog.yaml

@@ -6195,3 +6195,15 @@
     - "Use AITER INT4 quick-reduce, ptpc_fp8 online quantization excluding embeddings, lm_head, gates, and experts, an FP8 KV cache, and three-token MTP with golden synthetic acceptance length 2.99 for benchmark runs."
     - "Set max-num-seqs to twice the concurrency, select CUDA graph capture sizes by concurrency, and cap max-num-batched-tokens at 16384."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2576
+
+- config-keys:
+    - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Bump the DSv4 FP4 MI355X SGLang disagg AgentX HiCache MTP image from lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 to lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817."
+    - "Extend the MTP concurrency sweep: the TP8/EP1 no-DP arm to 1, 2, 4, 8, 16, 32, 48, and 64, and the TP8/EP8 DP-attention arm to 64, 96, 128, and 192, so concurrency 64 is measured on both topologies at the crossover."
+    - "models.yaml (DeepSeek-V4-Pro-AgentX): raise prefill mem_fraction_static from 0.72 to 0.85."
+    - "dsv4_fp4_mi355x_sglang-disagg.sh: lower the default HICACHE_RATIO from 4 to 3 (host pool = ratio * GPU KV pool), still pinned to the --hicache-ratio path through FORCE_HICACHE_RATIO."
+    - "AMD multi-node infrastructure, expected to be performance-neutral: job.slurm's pre-flight GPU drain gate now reuses wait_for_amd_gpu_clean from benchmark_lib.sh instead of the removed helpers/gpu_sanity.sh, and submit.sh resolves SLURM node exclusions per (framework, model) from node_excludes.yaml (Kimi-K2.5-MXFP4 on vllm-disagg only) with SLURM_EXCLUDE_NODES still overriding."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2643