Skip to content

[AMD] [AGENTX] Minimax-M3 Perf Tuning #2671

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ajith-sirra-amd wants to merge 3 commits into
base: main
Choose a base branch
from
+45 −105
Open

[AMD] [AGENTX] Minimax-M3 Perf Tuning #2671

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2fb48e1
[AMD] [AGENTX] Minimax-M3 Perf Tuning
AjithSirra Aug 19, 2026
7d31066
Merge branch 'main' into amd/minimax-m3-agentic-perf-tuning
ajith-sirra-amd Aug 19, 2026
3422737
[AMD] [AGENTX] Minimax-M3 Perf Tuning
AjithSirra Aug 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 36 additions & 104 deletions benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,21 @@
#!/usr/bin/env bash
set -eo pipefail
set -euo pipefail
set -x

# Agentic trace replay benchmark for MiniMax-M3 FP4 on MI355X using vLLM
# and EAGLE3 speculative decoding.
# EAGLE3 speculative decoding.
#
# Required env vars:
# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND,
# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING,
# TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION

source "$(dirname "$0")/../../benchmark_lib.sh"

# Force the eval framework to lm-eval for this recipe. run_eval derives its
# default as swebench for agentic scenarios (scenario_default=swebench when
# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence
# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so
# setting it here makes the effective framework always lm-eval, never swebench.
export EVAL_FRAMEWORK="lm-eval"

check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
Expand All @@ -23,16 +28,16 @@ NUM_SPEC_TOKENS=3
# minimax-m3.thinking_on[3]
SYNTHETIC_ACCEPT_LEN=2.78

if [[ -v SLURM_JOB_ID ]]; then
if [[ -n "${SLURM_JOB_ID+x}" ]]; then
echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
fi

# ROCR/HIP visibility for vLLM 0.14+
if [[ -v ROCR_VISIBLE_DEVICES ]]; then
if [[ -n "${ROCR_VISIBLE_DEVICES+x}" ]]; then
export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
fi

if [[ -n "$MODEL_PATH" ]]; then
if [[ -n "${MODEL_PATH:-}" ]]; then
if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
hf download "$MODEL" --local-dir "$MODEL_PATH"
fi
Expand All @@ -49,117 +54,51 @@ amd-smi || true
resolve_trace_source
install_agentic_deps

# Require the vLLM Prometheus stream in every official result. AIPerf
# deduplicates this endpoint against its automatic localhost discovery.
export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics"
export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:"

# ---- Server config ----------------------------------------------------------
SERVER_LOG="$RESULT_DIR/server.log"
LMCACHE_LOG="$RESULT_DIR/lmcache_server.log"
mkdir -p "$RESULT_DIR"

SERVER_PID=""
LMCACHE_PIDS=()
cleanup_agentic_services() {
local exit_code=$?
trap - EXIT INT TERM
set +e
stop_background_process_tree "$SERVER_PID" "vLLM server" 60
local i
for i in "${!LMCACHE_PIDS[@]}"; do
stop_background_process_tree "${LMCACHE_PIDS[$i]}" "LMCache server $i"
done
exit "$exit_code"
}
trap cleanup_agentic_services EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

# AgentX replays growing multi-turn prefixes, so keep prefix caching enabled
# for both GPU-resident and native-offload configurations.
OFFLOAD_ARGS=()

case "$KV_OFFLOAD_BACKEND" in
case "${KV_OFFLOAD_BACKEND:-}" in
"")
require_agentic_kv_offload_none
;;
lmcache)
require_agentic_kv_offload_backend lmcache

# Keep the image's tested torch/ROCm stack and install only LMCache's
# missing pure-Python runtime dependencies.
LMCACHE_VERSION="0.5.3"
LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-rocm"
agentic_pip_install --quiet --no-cache-dir --no-deps \
"sortedcontainers==2.4.0" \
"opentelemetry-exporter-prometheus==0.61b0" \
"cupy-rocm-7-0==14.1.1" \
"lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX"
python3 -c \
"import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \
>/dev/null

# Split the node-level DRAM limit evenly across one MP server per TP rank.
LMCACHE_N_SERVERS="$TP"
LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB"
SHM_FREE_GB=$(df -BG --output=avail /dev/shm 2>/dev/null | tail -1 | tr -dc '0-9')
if [ -n "$SHM_FREE_GB" ] && [ "$SHM_FREE_GB" -gt 0 ]; then
SHM_CAP_GB=$((SHM_FREE_GB * 90 / 100))
if [ "$LMCACHE_L1_SIZE_GB" -gt "$SHM_CAP_GB" ]; then
echo "Error: LMCache L1 ${LMCACHE_L1_SIZE_GB} GB exceeds 90% of free /dev/shm (${SHM_CAP_GB} GB)." >&2
exit 1
fi
fi
LMCACHE_L1_SHARD_GB=$((LMCACHE_L1_SIZE_GB / LMCACHE_N_SERVERS))
if [ "$LMCACHE_L1_SHARD_GB" -lt 1 ]; then
echo "Error: LMCache DRAM budget is less than 1 GB per TP rank." >&2
exit 1
fi

LMCACHE_SERVER_URLS=()
LMCACHE_HTTP_PORTS=()
LMCACHE_LOGS=()
: > "$RESULT_DIR/lmcache_command.txt"
for shard in $(seq 0 $((LMCACHE_N_SERVERS - 1))); do
shard_port=$((5555 + shard))
shard_http_port=$((8080 + shard))
shard_log="${LMCACHE_LOG%.log}_${shard}.log"
LMCACHE_CMD=(
lmcache server
--host 127.0.0.1
--port "$shard_port"
--http-host 127.0.0.1
--http-port "$shard_http_port"
--l1-size-gb "$LMCACHE_L1_SHARD_GB"
--l1-init-size-gb 10
--l1-read-ttl-seconds 7200
--chunk-size 256
--max-workers 2
--eviction-policy LRU
--supported-transfer-mode lmcache_driven
)
append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}"
"${LMCACHE_CMD[@]}" > "$shard_log" 2>&1 &
LMCACHE_PIDS+=($!)
LMCACHE_HTTP_PORTS+=("$shard_http_port")
LMCACHE_LOGS+=("$shard_log")
LMCACHE_SERVER_URLS+=("tcp://127.0.0.1:${shard_port}")
done
for shard in "${!LMCACHE_PIDS[@]}"; do
wait_for_ready \
--endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORTS[$shard]}/healthcheck" \
--log "${LMCACHE_LOGS[$shard]}" \
--pid "${LMCACHE_PIDS[$shard]}" \
--sleep-interval 1 \
--timeout 600
done
LMCACHE_SERVER_URLS_CSV=$(IFS=,; echo "${LMCACHE_SERVER_URLS[*]}")
vllm-native)
require_agentic_kv_offload_backend vllm-native
unset VLLM_USE_SIMPLE_KV_OFFLOAD
# Use vLLM's regular native KV-offload path (OffloadingConnector),
# NOT the SimpleCPUOffloadConnector. The "vllm-native" backend resolves to
# OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1
# would switch it to SimpleCPUOffloadConnector. We intentionally leave
# that env var UNSET here so the regular OffloadingConnector path is
# used. The shortcut --kv_offloading_backend native + --kv_offloading_size
# form constructs the KVTransferConfig at engine startup
# (vllm/config/vllm.py:662).

# Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default)
# This gives extra cache hit than disabling hybrid kv cache manager
OFFLOAD_ARGS=(
--kv-transfer-config
"{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.server_urls\":\"$LMCACHE_SERVER_URLS_CSV\",\"lmcache.mp.mq_timeout\":6000.0}}"
--kv_offloading_backend native
--kv_offloading_size "$TOTAL_CPU_DRAM_GB"
)
;;
*)
echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND (expected empty or lmcache)" >&2
echo "Unsupported KV_OFFLOAD_BACKEND: ${KV_OFFLOAD_BACKEND:-}" >&2
exit 1
;;
esac
Comment on lines 95 to 104

@claude claude Bot Aug 19, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔴 The script's KV_OFFLOAD_BACKEND case statement only handles "" and "vllm-native" now (the lmcache branch was removed), but configs/amd-master.yaml's minimaxm3-fp4-mi355x-vllm-agentic-mtp entry still has a third search-space arm using kv-offload-backend: { name: lmcache, version: "0.5.3" } (conc-list [32, 40]). That arm will set KV_OFFLOAD_BACKEND=lmcache, hit the script's *) default branch, and exit 1 with "Unsupported KV_OFFLOAD_BACKEND: lmcache", deterministically failing those two sweep points.

Extended reasoning...

The bug

This PR rewrites benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh to drop LMCache support entirely and replace it with a vllm-native backend (using --kv_offloading_backend native / --kv_offloading_size). The case "${KV_OFFLOAD_BACKEND:-}" statement now only has two real branches: "" (no offload) and vllm-native. Everything else — including the previously-supported lmcache value — falls through to the *) branch, which prints Unsupported KV_OFFLOAD_BACKEND: ... and exit 1.

The triggering config

configs/amd-master.yaml (unchanged by this PR, at line 1701) still has a third search-space arm for minimaxm3-fp4-mi355x-vllm-agentic-mtp:

- { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [32, 40], spec-decoding: mtp }

The code path that triggers it

  1. .github/workflows/run-sweep.yml sets the workflow input kv-offload-backend: ${{ matrix.config["kv-offload-backend"].name }}, which for this arm resolves to the literal string "lmcache".
  2. .github/workflows/benchmark-tmpl.yml exports this as KV_OFFLOAD_BACKEND=lmcache into the job environment.
  3. The rewritten script's case statement no longer has an lmcache) branch (it was entirely deleted in this diff, replaced by vllm-native)), so KV_OFFLOAD_BACKEND=lmcache falls into *).
  4. The script prints Unsupported KV_OFFLOAD_BACKEND: lmcache to stderr and calls exit 1.

Why nothing else catches this

There's no validation step between the YAML config and the shell script that cross-checks backend names against what the script actually supports — utils/matrix_logic only validates that kv-offload-backend.name is a non-empty string with an optional version (see test_kv_offload_backend_accepts_optional_version), so lmcache passes that check and reaches the runner, where it fails at execution time instead of at config-validation time.

Step-by-step proof

  1. Config arm: tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [32, 40].
  2. Sweep matrix expansion produces two jobs (conc=32, conc=40) with KV_OFFLOAD_BACKEND=lmcache set in the environment.
  3. minimaxm3_fp4_mi355x_mtp.sh runs case "${KV_OFFLOAD_BACKEND:-}" in "") ... ;; vllm-native) ... ;; *) echo "Unsupported KV_OFFLOAD_BACKEND: lmcache" >&2; exit 1 ;; esac.
  4. Since lmcache matches neither "" nor vllm-native, the *) branch runs, and the script exits 1 before ever starting the vLLM server.
  5. Both sweep points (conc 32 and conc 40) for that arm fail deterministically on every run — this is not a flake.

Before this PR, the script had a working lmcache) branch, so this is a regression introduced specifically by this diff, not a pre-existing issue. The PR description itself only mentions "vllm-simple" DRAM offloading, so the author likely didn't intend to keep the config's lmcache arm active — that arm should either be updated to use vllm-native (matching the new script) or removed if lmcache support is being fully retired for this recipe.

Fix

Either:

  • Update the lmcache arm in configs/amd-master.yaml to use kv-offload-backend: { name: vllm-native } (dropping the version field, which is no longer needed), or
  • Restore an lmcache) branch in the script if LMCache support should be kept alongside the new vllm-native path.

Expand Down Expand Up @@ -187,14 +126,7 @@ export VLLM_USE_BREAKABLE_CUDAGRAPH=0
export VLLM_ROCM_USE_AITER=1
export VLLM_ROCM_USE_AITER_MOE=1
export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1
# The AITER page-16 sparse-attention path requires exactly one KV head per
# tensor-parallel rank. MiniMax-M3 has four KV heads, so TP4 uses that fast
# path while TP2 uses vLLM's supported Triton sparse-attention fallback.
if [ "$TP" -eq 4 ]; then
export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1
else
export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0
fi
export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1
Comment on lines 126 to +129

@claude claude Bot Aug 19, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔴 Removing the TP-conditional guard on VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT and hard-coding it to 1 violates the AITER page-16 fast path's documented requirement of exactly one KV head per TP rank (MiniMax-M3 has 4 KV heads). configs/amd-master.yaml's minimaxm3-fp4-mi355x-vllm-agentic-mtp still has a live { tp: 2, ... } arm, which will now launch with 2 heads/rank under the shuffled layout — a topology the deleted comment explicitly says only supports the Triton fallback (shuffle=0). Restore the if [ "$TP" -eq 4 ] guard, or confirm the new v0.27.1 image no longer needs it and drop the TP2 arm instead.

Extended reasoning...

The bug: The previous version of this script conditioned VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT on the tensor-parallel size, with a comment explaining why:

# The AITER page-16 sparse-attention path requires exactly one KV head per
# tensor-parallel rank. MiniMax-M3 has four KV heads, so TP4 uses that fast
# path while TP2 uses vLLM'\''s supported Triton sparse-attention fallback.
if [ "$TP" -eq 4 ]; then
    export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1
else
    export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0
fi

This PR deletes both the comment and the conditional, replacing them with a single unconditional line (benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh:126-129):

export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1

Why this matters here specifically: MiniMax-M3 has 4 KV heads. The deleted comment states the AITER page-16 sparse-attention fast path (which this shuffle env var enables) requires exactly 1 KV head per TP rank. At TP=4, that's 4 heads / 4 ranks = 1 head/rank — valid. At TP=2, that's 4 heads / 2 ranks = 2 heads/rank — the documented invariant is violated, and the recipe author's own fallback (shuffle=0, Triton path) exists precisely to handle that case.

The trigger path: configs/amd-master.yaml (also touched by this PR, just above the diff hunk shown) still defines a live TP2 arm for this exact recipe:

minimaxm3-fp4-mi355x-vllm-agentic-mtp:
  ...
  search-space:
  - { tp: 4, kv-offloading: none, conc-list: [...], spec-decoding: mtp }
  - { tp: 2, kv-offloading: none, conc-list: [1, 2, 5], spec-decoding: mtp }
  - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [32, 40], spec-decoding: mtp }

Every sweep point in the tp: 2 arm launches minimaxm3_fp4_mi355x_mtp.sh with TP=2, which after this change unconditionally sets VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1. There is nothing else in the script that neutralizes this for TP2 — --attention-backend TRITON_ATTN was already present in the old script alongside the TP-conditional, so it does not make the shuffle var redundant; the shuffle var governs the KV-cache physical layout independently (relevant given the --hf-overrides '{"text_config": {"use_index_cache": true, "index_topk_freq": 4}}' MSA/index-cache path also present in this script).

Why nothing else in this PR prevents it: The PR is otherwise a broad refactor (bash strict-mode fixes, swapping LMCache for vLLM-native KV offload, dropping AIPerf metrics vars, etc.) and the image was bumped to vllm-openai-rocm:v0.27.1. Nothing in the diff or PR description states that the new image relaxes the page-16 fast path's one-head-per-rank requirement — this looks like an accidental drop of a still-necessary guard rather than an intentional simplification, since the guard is a hardware/algorithm invariant tied to MiniMax-M3'''s head count, not an image-version workaround.

Step-by-step proof:

  1. configs/amd-master.yaml schedules a sweep point with tp: 2 for minimaxm3-fp4-mi355x-vllm-agentic-mtp.
  2. The launcher sets TP=2 and invokes minimaxm3_fp4_mi355x_mtp.sh.
  3. The script unconditionally executes export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 (line 129).
  4. vllm serve starts with --tensor-parallel-size 2 and the AITER page-16 shuffled KV-cache layout enabled.
  5. MiniMax-M3'''s 4 KV heads split across 2 ranks give 2 heads/rank, which the deleted comment states is unsupported by that fast path (only 1 head/rank is supported); the TP4 arm in the same config is the only one satisfying that invariant.
  6. Depending on how strictly AITER validates this at engine startup, this either fails to launch or produces a silently incorrect KV-cache layout for the TP2 sweep points — either way, those 3 concurrency points (1, 2, 5) in the search space are broken by this change.

The fix: Restore the TP-conditional (if [ "$TP" -eq 4 ]; then ... else ... fi) exactly as it was, or — if the v0.27.1 image genuinely no longer needs the one-head-per-rank restriction — state that explicitly and verify it, since dropping the check silently while keeping the TP2 arm risks a broken or corrupted sweep point either way.

export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4
export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0
export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256
Expand All @@ -218,13 +150,14 @@ VLLM_CMD=(
--tool-call-parser minimax_m3
--enable-auto-tool-choice
--default-chat-template-kwargs '{"thinking_mode":"enabled"}'
--max-num-seqs "$((2 * CONC))"
--max-num-seqs "$CONC"
--stream-interval 20
--hf-overrides '{"text_config": {"use_index_cache": true, "index_topk_freq": 4}}'
--speculative-config "$SPEC_CONFIG"
"${OFFLOAD_ARGS[@]}"
)
write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}"
printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
SERVER_PID=$!
echo "Server PID: $SERVER_PID"
Expand All @@ -236,6 +169,5 @@ if [ "${EVAL_ONLY}" = "true" ]; then
run_eval --port "$PORT"
else
build_replay_cmd "$RESULT_DIR"
REPLAY_CMD+=" --apply-chat-template"
run_agentic_replay_and_write_outputs "$RESULT_DIR"
fi
fi
2 changes: 1 addition & 1 deletion configs/amd-master.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1628,7 +1628,7 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp:
multinode: false
scenarios:
agentic-coding:
- dram-utilization: 0.20
- dram-utilization: 0.50
search-space:
- { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 5, 8, 10, 12, 15, 20, 24, 28, 32], spec-decoding: mtp }
- { tp: 2, kv-offloading: none, conc-list: [1, 2, 5], spec-decoding: mtp }
Expand Down
8 changes: 8 additions & 0 deletions perf-changelog.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6195,3 +6195,11 @@
- "Use AITER INT4 quick-reduce, ptpc_fp8 online quantization excluding embeddings, lm_head, gates, and experts, an FP8 KV cache, and three-token MTP with golden synthetic acceptance length 2.99 for benchmark runs."
- "Set max-num-seqs to twice the concurrency, select CUDA graph capture sizes by concurrency, and cap max-num-batched-tokens at 16384."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2576

- config-keys:
- minimaxm3-fp4-mi355x-vllm-agentic-mtp
scenario-type:
- agentic-coding
description:
- "Add the MI355X Minimax-M3 FP4 Agentic MTP"
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2671