[AMD] [WIP] [AGENTX] GLM-5.2 MTP5 tune: raise acc-len and drop hicache at low conc / GLM-5.2 MTP5 调优：提升接受率目标并在低并发下关闭 hicache

benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh

@@ -184,22 +184,27 @@ if [ "$DP_ATTENTION" = "true" ]; then
     export SGLANG_DP_USE_REDUCE_SCATTER=1
     export GPU_MAX_HW_QUEUES=5
 elif [ "$CONC" -le 16 ]; then
-    # A full 131072-token prefill chunk needs ~7 GiB/rank of activation
-    # headroom on top of the static pool; pair it with mem-fraction 0.80
-    # like the FP8 sibling's low-conc band (0.85 OOMs the device mid-replay:
+    # Chunked prefill 32k: smaller chunks let the scheduler interleave decode
+    # steps between prefill chunks, reducing TPOT for concurrent sessions
+    # (improved interactivity vs the original 131072-token chunk). The reduced
+    # chunk size drops per-chunk activation headroom from ~7 GiB/rank to
+    # ~1.7 GiB/rank, so mem-fraction 0.85 is safe (0.85 OOMed at 131k:
     # "Tried to allocate 6.86 GiB ... 5.15 GiB is free", run 29751563205).
-    CHUNKED_PREFILL_SIZE=131072
-    MEM_FRACTION_STATIC=0.80
+    CHUNKED_PREFILL_SIZE=32768
+    MEM_FRACTION_STATIC=0.85
 else
     CHUNKED_PREFILL_SIZE=32768
     export AGENTIC_WARMUP_GRACE_PERIOD=3600
 fi
 MAX_RUNNING_REQUESTS=$((1 * CONC))
 [ "$MAX_RUNNING_REQUESTS" -gt 256 ] && MAX_RUNNING_REQUESTS=256
 CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS
+# NOTE: with MTP num-steps=5 the draft+verify batch can momentarily exceed
+# MAX_RUNNING_REQUESTS; if cuda-graph misses ("graph capture miss") appear in
+# server.log under load, consider raising this to e.g. MAX_RUNNING_REQUESTS * 2.
 
 if [ "${EVAL_ONLY:-false}" != "true" ]; then
-    export SGLANG_SIMULATE_ACC_LEN=2.99
+    export SGLANG_SIMULATE_ACC_LEN=3.61
     export SGLANG_SIMULATE_ACC_METHOD=match-expected
     export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token
 fi
@@ -225,9 +230,9 @@ SGLANG_CMD=(
     --max-running-requests "$MAX_RUNNING_REQUESTS"
     --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
     --speculative-algorithm EAGLE
-    --speculative-num-steps 3
+    --speculative-num-steps 5
     --speculative-eagle-topk 1
-    --speculative-num-draft-tokens 4
+    --speculative-num-draft-tokens 6
     "${CACHE_ARGS[@]}"
     --watchdog-timeout 1800
     --enable-metrics

configs/amd-master.yaml

@@ -1650,8 +1650,9 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp:
       - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [32, 40], spec-decoding: mtp }
 
 # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative
-# decoding. TP=4 EP=4 with KV offloading to DRAM (hicache backend) to support
-# long agentic context windows. Concurrency sweep [1, 2, 4, 8, 10].
+# decoding. Two arms: (1) TP4/EP4 with HiCache KV offloading to DRAM, concurrency
+# sweep [1, 2, 4, 8, 10, 12, 16]; (2) TP8/EP8 without KV offloading for low-latency
+# comparison at conc [1, 2, 4]. SA selects the Pareto-optimal arm per concurrency.
 glm5.2-fp4-mi355x-sglang-agentic-mtp:
   image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728
   model: amd/GLM-5.2-MXFP4
@@ -1665,4 +1666,4 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp:
     - dram-utilization: 0.8
       search-space:
       - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp }
-      - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4], spec-decoding: mtp }
+      - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp }

perf-changelog.yaml

@@ -5837,3 +5837,10 @@
     - "Collect and require SGLang Prometheus metrics from the aggregate engine endpoint"
     - "Image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2563
+
+- config-keys:
+    - glm5.2-fp4-mi355x-sglang-agentic-mtp
+  description:
+    - "Increase MTP speculative steps from 3 to 5 (num-draft-tokens 4→6) and raise SGLANG_SIMULATE_ACC_LEN from 2.99 to 3.61 to reflect higher measured acceptance rate, targeting ~15% throughput improvement"
+    - "Replace the TP8/EP8 low-concurrency arm (conc [1,2,4]) hicache offload with kv-offloading: none to reduce per-request latency at low load; conc [1,2,4] are now tested on both tp=4+hicache and tp=8+no-offload so SemiAnalysis can select the Pareto-optimal point per concurrency"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2570