diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index d7ed7a359..98b4531e4 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -184,12 +184,14 @@ if [ "$DP_ATTENTION" = "true" ]; then export SGLANG_DP_USE_REDUCE_SCATTER=1 export GPU_MAX_HW_QUEUES=5 elif [ "$CONC" -le 16 ]; then - # A full 131072-token prefill chunk needs ~7 GiB/rank of activation - # headroom on top of the static pool; pair it with mem-fraction 0.80 - # like the FP8 sibling's low-conc band (0.85 OOMs the device mid-replay: + # Chunked prefill 32k: smaller chunks let the scheduler interleave decode + # steps between prefill chunks, reducing TPOT for concurrent sessions + # (improved interactivity vs the original 131072-token chunk). The reduced + # chunk size drops per-chunk activation headroom from ~7 GiB/rank to + # ~1.7 GiB/rank, so mem-fraction 0.85 is safe (0.85 OOMed at 131k: # "Tried to allocate 6.86 GiB ... 5.15 GiB is free", run 29751563205). - CHUNKED_PREFILL_SIZE=131072 - MEM_FRACTION_STATIC=0.80 + CHUNKED_PREFILL_SIZE=32768 + MEM_FRACTION_STATIC=0.85 else CHUNKED_PREFILL_SIZE=32768 export AGENTIC_WARMUP_GRACE_PERIOD=3600 @@ -197,9 +199,12 @@ fi MAX_RUNNING_REQUESTS=$((1 * CONC)) [ "$MAX_RUNNING_REQUESTS" -gt 256 ] && MAX_RUNNING_REQUESTS=256 CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS +# NOTE: with MTP num-steps=5 the draft+verify batch can momentarily exceed +# MAX_RUNNING_REQUESTS; if cuda-graph misses ("graph capture miss") appear in +# server.log under load, consider raising this to e.g. MAX_RUNNING_REQUESTS * 2. if [ "${EVAL_ONLY:-false}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=2.99 + export SGLANG_SIMULATE_ACC_LEN=3.61 export SGLANG_SIMULATE_ACC_METHOD=match-expected export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token fi @@ -225,9 +230,9 @@ SGLANG_CMD=( --max-running-requests "$MAX_RUNNING_REQUESTS" --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" --speculative-algorithm EAGLE - --speculative-num-steps 3 + --speculative-num-steps 5 --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 6 "${CACHE_ARGS[@]}" --watchdog-timeout 1800 --enable-metrics diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 07f3defd9..c94029484 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1701,8 +1701,9 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [32, 40], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative -# decoding. TP=4 EP=4 with KV offloading to DRAM (hicache backend) to support -# long agentic context windows. Concurrency sweep [1, 2, 4, 8, 10]. +# decoding. Two arms: (1) TP4/EP4 with HiCache KV offloading to DRAM, concurrency +# sweep [1, 2, 4, 8, 10, 12, 16]; (2) TP8/EP8 without KV offloading for low-latency +# comparison at conc [1, 2, 4]. SA selects the Pareto-optimal arm per concurrency. glm5.2-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 model: amd/GLM-5.2-MXFP4 @@ -1716,4 +1717,4 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: - dram-utilization: 0.8 search-space: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4], spec-decoding: mtp } + - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 676fc73e7..f118b99aa 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5902,3 +5902,10 @@ - "Use NIXL for KV transfer with EAGLE speculative decoding and chat-formatted benchmark inputs." - "Image: lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2554 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + description: + - "Increase MTP speculative steps from 3 to 5 (num-draft-tokens 4→6) and raise SGLANG_SIMULATE_ACC_LEN from 2.99 to 3.61 to reflect higher measured acceptance rate, targeting ~15% throughput improvement" + - "Replace the TP8/EP8 low-concurrency arm (conc [1,2,4]) hicache offload with kv-offloading: none to reduce per-request latency at low load; conc [1,2,4] are now tested on both tp=4+hicache and tp=8+no-offload so SemiAnalysis can select the Pareto-optimal point per concurrency" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2570