From 203d594081c15883a293456d9aba3dc52d37d02f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 16:55:29 -0500 Subject: [PATCH 1/5] [AMD][AgentX] Add MI325X Qwen3.5 FP8 SGLang MTP --- .../agentic/qwen3.5_fp8_mi325x_mtp.sh | 149 ++++++++++++++++++ configs/amd-master.yaml | 23 ++- perf-changelog.yaml | 10 ++ 3 files changed, 181 insertions(+), 1 deletion(-) create mode 100755 benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh new file mode 100755 index 000000000..f16c0526f --- /dev/null +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# AgentX trace replay for Qwen3.5-397B-A17B FP8 on MI325X with SGLang +# native EAGLE/NEXTN MTP. Throughput uses the committed golden synthetic +# acceptance length; evaluation retains real target-model verification. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars \ + MODEL TP CONC EP_SIZE KV_OFFLOADING \ + TOTAL_CPU_DRAM_GB RESULT_DIR DURATION + +SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +rocm-smi || true +amd-smi || true + +export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k +resolve_trace_source +install_agentic_deps + +# This is a single aggregate SGLang engine, so one logical backend metrics +# endpoint is authoritative. build_replay_cmd also discovers the public +# endpoint; AIPerf deduplicates the explicit copy. +export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "SGLang server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +if agentic_kv_offload_enabled; then + echo "Error: the initial MI325X discovery pass supports GPU-resident KV only." >&2 + exit 1 +fi + +PARALLEL_ARGS=( + --tp "$TP" + --dp 1 + --ep-size "$EP_SIZE" +) + +TOKENIZER_ARGS=() +if [ "$TP" -ge 4 ]; then + TOKENIZER_ARGS=(--tokenizer-worker-num 6) +fi + +# AgentX concurrency counts live session trees rather than HTTP requests. +# Keep capacity for subagent fan-out while bounding graph-capture memory. +MAX_RUNNING_REQUESTS=$((2 * CONC)) +CUDA_GRAPH_MAX_BS="$CONC" +[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 + +export PYTHONNOUSERSITE=1 +export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 +export SGLANG_USE_AITER=1 +export SGLANG_USE_AITER_UNIFIED_ATTN=1 +export SGLANG_TIMEOUT_KEEP_ALIVE=1800 + +# Synthetic rejection sampling is only for performance replay. The AL is the +# committed Qwen3.5 thinking-on value for three speculative tokens. Evals use +# real target-model verification. +if [ "${EVAL_ONLY:-false}" != "true" ]; then + export SGLANG_SIMULATE_ACC_LEN=3.39 + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token +fi + +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --trust-remote-code + "${PARALLEL_ARGS[@]}" + --attention-backend aiter + --quantization fp8 + --kv-cache-dtype fp8_e4m3 + --mamba-ssm-dtype bfloat16 + --mem-fraction-static 0.80 + --model-loader-extra-config '{"enable_multithread_load": true}' + --watchdog-timeout 1200 + --enable-aiter-allreduce-fusion + --page-size 16 + --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --max-prefill-tokens 32768 + --chunked-prefill-size 32768 + --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" + --stream-interval 50 + "${TOKENIZER_ARGS[@]}" + --tokenizer-path "$MODEL" + --reasoning-parser qwen3 + --tool-call-parser qwen3_coder + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 + --enable-metrics + --enable-cache-report +) + +printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" +printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" +"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "${EVAL_ONLY:-false}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --use-chat-template" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9bc224b16..443e210f9 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -228,6 +228,28 @@ qwen3.5-fp8-mi355x-sglang-agentic: - search-space: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } +# MI325X fast-discovery grid for Qwen3.5's native one-layer MTP head. The +# model is 378 GiB on disk, so TP2 is the smallest plausible serving shape on +# 256 GiB GPUs. TP4/TP8 provide progressively more KV capacity; matching EP +# arms test whether expert sharding repays its communication cost on xGMI. +qwen3.5-fp8-mi325x-sglang-agentic-mtp: + image: lmsysorg/sglang:v0.5.16-rocm720-mi30x + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:mi325x-amds + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64] } + - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 64, 80, 96] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 8, 12, 16, 24, 32, 40, 48, 64] } + qwen3.5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -1633,4 +1655,3 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: search-space: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4], spec-decoding: mtp } - diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0bce8d894..dc61282ee 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5818,3 +5818,13 @@ description: - "Extend the SimpleCPUOffloadConnector grid to c8/c12/c16/c20/c24/c28/c32/c48/c64 to locate its crossover against the resident curve" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 + +- config-keys: + - qwen3.5-fp8-mi325x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MI325X Qwen3.5 FP8 AgentX with SGLang native EAGLE MTP and golden synthetic acceptance length 3.39" + - "Use lmsysorg/sglang:v0.5.16-rocm720-mi30x, FP8 KV cache, AITER attention, and required SGLang Prometheus metrics" + - "Select the official TP2/EP2, TP4, TEP4, TP8, and TEP8 matrix from the complete 62-point fast discovery sweep" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2566 From 6b48e18eb6a54bcf63941ed546b79646c10fad22 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 16:56:47 -0500 Subject: [PATCH 2/5] Fix AgentX chat-template flag and changelog link --- benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh index f16c0526f..bfe04db6b 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh @@ -144,6 +144,6 @@ if [ "${EVAL_ONLY:-false}" = "true" ]; then run_eval --port "$PORT" else build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --use-chat-template" + REPLAY_CMD+=" --apply-chat-template" run_agentic_replay_and_write_outputs "$RESULT_DIR" fi From 4112d4205d72b9634938fb37bdee26d790ec35ba Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 17:03:59 -0500 Subject: [PATCH 3/5] fix(amd): fit qwen3.5 mtp hybrid cache on tp2 --- .../single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh index bfe04db6b..556aa5a3f 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh @@ -83,6 +83,14 @@ MAX_RUNNING_REQUESTS=$((2 * CONC)) CUDA_GRAPH_MAX_BS="$CONC" [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 +# With EAGLE enabled, SGLang reserves 0.12 of GPU memory for the draft model. +# TP2 needs the remaining target-model budget raised from 0.68 to 0.78 to fit +# Qwen3.5's hybrid state cache; TP4/TP8 retain the established 0.68 budget. +MEM_FRACTION_STATIC=0.80 +if [ "$TP" -eq 2 ]; then + MEM_FRACTION_STATIC=0.90 +fi + export PYTHONNOUSERSITE=1 export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 export SGLANG_USE_AITER=1 @@ -110,7 +118,7 @@ SGLANG_CMD=( --quantization fp8 --kv-cache-dtype fp8_e4m3 --mamba-ssm-dtype bfloat16 - --mem-fraction-static 0.80 + --mem-fraction-static "$MEM_FRACTION_STATIC" --model-loader-extra-config '{"enable_multithread_load": true}' --watchdog-timeout 1200 --enable-aiter-allreduce-fusion From 2f257a987403769562ab942653dd06c08c83efbc Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 17:42:34 -0500 Subject: [PATCH 4/5] fix(amd): reserve tp2 hybrid and kv pools --- benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh index 556aa5a3f..c7b78e3a9 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh @@ -83,12 +83,12 @@ MAX_RUNNING_REQUESTS=$((2 * CONC)) CUDA_GRAPH_MAX_BS="$CONC" [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 -# With EAGLE enabled, SGLang reserves 0.12 of GPU memory for the draft model. -# TP2 needs the remaining target-model budget raised from 0.68 to 0.78 to fit -# Qwen3.5's hybrid state cache; TP4/TP8 retain the established 0.68 budget. +# With EAGLE enabled, SGLang reserves 15% of this budget for the draft model. +# TP2 needs an effective 0.8075 target-model budget to fit Qwen3.5's weights, +# hybrid state, and KV pools; TP4/TP8 retain the established 0.68 budget. MEM_FRACTION_STATIC=0.80 if [ "$TP" -eq 2 ]; then - MEM_FRACTION_STATIC=0.90 + MEM_FRACTION_STATIC=0.95 fi export PYTHONNOUSERSITE=1 From ada0d27092e2bfde02ebb33af8b8fa3e868842bd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 22:51:43 -0500 Subject: [PATCH 5/5] Tune MI325X Qwen3.5 AgentX sweep --- benchmarks/benchmark_lib.sh | 8 ++++++++ .../agentic/qwen3.5_fp8_mi325x_mtp.sh | 12 +++--------- configs/amd-master.yaml | 17 ++++++++--------- perf-changelog.yaml | 2 +- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 4e2e7b7b1..87adc8d98 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -420,6 +420,14 @@ wait_for_server_ready() { kill $TAIL_PID } +# Persist an argv array in shell-replayable form. +write_command() { + local output_file="$1" + shift + printf '%q ' "$@" | tee "$output_file" + printf '\n' | tee -a "$output_file" +} + # Run benchmark serving with standardized parameters # All parameters are required except --endpoint, --use-chat-template, --dsv4, and --trust-remote-code # Parameters: diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh index c7b78e3a9..123e1c4ac 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -euo pipefail +set -eo pipefail set -x # AgentX trace replay for Qwen3.5-397B-A17B FP8 on MI325X with SGLang @@ -11,7 +11,7 @@ source "$(dirname "$0")/../../benchmark_lib.sh" export EVAL_FRAMEWORK="lm-eval" check_env_vars \ - MODEL TP CONC EP_SIZE KV_OFFLOADING \ + MODEL TP CONC EP_SIZE \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} @@ -61,11 +61,6 @@ trap cleanup_agentic_services EXIT trap 'exit 130' INT trap 'exit 143' TERM -if agentic_kv_offload_enabled; then - echo "Error: the initial MI325X discovery pass supports GPU-resident KV only." >&2 - exit 1 -fi - PARALLEL_ARGS=( --tp "$TP" --dp 1 @@ -141,8 +136,7 @@ SGLANG_CMD=( --enable-cache-report ) -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" +write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" "${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 443e210f9..6e83ee66f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -228,10 +228,9 @@ qwen3.5-fp8-mi355x-sglang-agentic: - search-space: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } -# MI325X fast-discovery grid for Qwen3.5's native one-layer MTP head. The -# model is 378 GiB on disk, so TP2 is the smallest plausible serving shape on -# 256 GiB GPUs. TP4/TP8 provide progressively more KV capacity; matching EP -# arms test whether expert sharding repays its communication cost on xGMI. +# MI325X official matrix selected from the complete 62-point fast sweep. TP2 +# peaks at c4, TP4/TEP4 at c40, and TP8/TEP8 at c64; the next point beyond each +# knee is retained to document the throughput cliff. qwen3.5-fp8-mi325x-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-rocm720-mi30x model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -244,11 +243,11 @@ qwen3.5-fp8-mi325x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64] } - - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64] } - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 64, 80, 96] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 8, 12, 16, 24, 32, 40, 48, 64] } + - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 6] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 8, 16, 32, 40, 48] } + - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 32, 40, 48] } + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 24, 32, 40, 48, 64, 80] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 24, 32, 40, 48, 64] } qwen3.5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index dc61282ee..c6bf5ef4a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5826,5 +5826,5 @@ description: - "Add MI325X Qwen3.5 FP8 AgentX with SGLang native EAGLE MTP and golden synthetic acceptance length 3.39" - "Use lmsysorg/sglang:v0.5.16-rocm720-mi30x, FP8 KV cache, AITER attention, and required SGLang Prometheus metrics" - - "Select the official TP2/EP2, TP4, TEP4, TP8, and TEP8 matrix from the complete 62-point fast discovery sweep" + - "Select a 32-point official TP2/EP2, TP4, TEP4, TP8, and TEP8 matrix from the complete 62-point fast discovery sweep" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2566