[AMD] [AgentX] Add MI300X Qwen3.5 FP8 SGLang MTP

benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+set -eo pipefail
+set -x
+
+# Qwen3.5-397B-A17B FP8 on MI300X with native EAGLE/NEXTN MTP.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+export EVAL_FRAMEWORK="lm-eval"
+
+check_env_vars \
+    MODEL TP CONC EP_SIZE KV_OFFLOADING \
+    RESULT_DIR DURATION
+
+require_agentic_kv_offload_none
+
+SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30}
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
+fi
+
+if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then
+    export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
+fi
+
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+else
+    hf download "$MODEL"
+    export MODEL_PATH="$MODEL"
+fi
+rocm-smi || true
+amd-smi || true
+
+export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k
+resolve_trace_source
+install_agentic_deps
+
+SERVER_LOG="$RESULT_DIR/server.log"
+mkdir -p "$RESULT_DIR"
+
+PARALLEL_ARGS=(
+    --tensor-parallel-size "$TP"
+    --data-parallel-size 1
+    --ep-size "$EP_SIZE"
+)
+
+# AgentX concurrency counts live session trees, not individual HTTP requests.
+MAX_RUNNING_REQUESTS=$((2 * CONC))
+CUDA_GRAPH_MAX_BS="$CONC"
+[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64
+
+export PYTHONNOUSERSITE=1
+export SGLANG_ENABLE_SPEC_V2=1
+export SGLANG_TIMEOUT_KEEP_ALIVE=1800
+
+if [ "${EVAL_ONLY:-false}" != "true" ]; then
+    export SGLANG_SIMULATE_ACC_LEN=3.39
+    export SGLANG_SIMULATE_ACC_METHOD=match-expected
+    export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token
+fi
+
+SGLANG_CMD=(
+    python3 -m sglang.launch_server
+    --model-path "$MODEL_PATH"
+    --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$PORT"
+    --trust-remote-code
+    "${PARALLEL_ARGS[@]}"
+    --attention-backend aiter
+    --enable-aiter-allreduce-fusion
+    --mamba-ssm-dtype bfloat16
+    --tokenizer-worker-num 6
+    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
+    --max-prefill-tokens 32768
+    --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
+    --mem-fraction-static 0.75
+    --tokenizer-path "$MODEL"
+    --reasoning-parser qwen3
+    --tool-call-parser qwen3_coder
+    --speculative-algorithm EAGLE
+    --speculative-num-steps 3
+    --speculative-eagle-topk 1
+    --speculative-num-draft-tokens 4
+    --enable-metrics
+    --enable-cache-report
+)
+
+printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
+"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+if [ "${EVAL_ONLY:-false}" = "true" ]; then
+    run_eval --port "$PORT"
+else
+    # Aggregate serving exposes one logical SGLang Prometheus target.
+    export AIPERF_SERVER_METRICS_URLS="http://localhost:$PORT/metrics"
+    export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:"
+    build_replay_cmd "$RESULT_DIR"
+    run_agentic_replay_and_write_outputs "$RESULT_DIR"
+fi

configs/amd-master.yaml

@@ -402,6 +402,23 @@ qwen3.5-fp8-mi300x-sglang:
       search-space:
       - { tp: 8, conc-start: 4, conc-end: 64 }
 
+# AgentX Pareto sweep for Qwen3.5 FP8 on MI300X. The fast discovery run peaks
+# at c24; c32 records the post-knee throughput and latency cliff.
+qwen3.5-fp8-mi300x-sglang-agentic-mtp:
+  image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730
+  model: Qwen/Qwen3.5-397B-A17B-FP8
+  model-prefix: qwen3.5
+  runner: cluster:mi300x-amds
+  precision: fp8
+  framework: sglang
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32] }
+      - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32] }
+
 dsr1-fp8-mi355x-atom:
   image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511
   model: deepseek-ai/DeepSeek-R1-0528
@@ -1633,4 +1650,3 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp:
       search-space:
       - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp }
       - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4], spec-decoding: mtp }
-

perf-changelog.yaml

@@ -5818,3 +5818,15 @@
   description:
     - "Extend the SimpleCPUOffloadConnector grid to c8/c12/c16/c20/c24/c28/c32/c48/c64 to locate its crossover against the resident curve"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475
+
+- config-keys:
+    - qwen3.5-fp8-mi300x-sglang-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Add Qwen3.5-397B-A17B FP8 AgentX on MI300X with SGLang native EAGLE MTP and golden synthetic acceptance length 3.39"
+    - "Use TP8/EP1 and TP8/EP8 Pareto sweeps at c4/c16/c20/c24/c32; fast discovery found peak throughput at c24 and a post-knee regression at c32"
+    - "Exclude HiCache because SGLang v0.5.16 rejects Qwen3.5's layer-first hybrid cache layout"
+    - "Collect and require SGLang Prometheus metrics from the aggregate engine endpoint"
+    - "Image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2563

runners/launch_mi300x-amds.sh

@@ -1,12 +1,17 @@
 #!/usr/bin/env bash
-set -eo pipefail
+set -euo pipefail
 
 export HF_HUB_CACHE_MOUNT="/raid/hf-hub-cache/"
-export PORT=8888
 
 PARTITION="compute"
-SQUASH_FILE="/home/gharunner/gharunners/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
+SQUASH_DIR="/raid/hf-hub-cache/runtime-cache/enroot"
+SQUASH_FILE="$SQUASH_DIR/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
 LOCK_FILE="${SQUASH_FILE}.lock"
+COMPUTE_TMPDIR="$SQUASH_DIR/tmp-${UID}"
+# Some MI300X compute images do not provision the login user's /run/user/$UID
+# directory. Keep Enroot's runtime state node-local instead of inheriting the
+# controller's XDG_RUNTIME_DIR, which is not valid on those compute nodes.
+export XDG_RUNTIME_DIR="/tmp/enroot-runtime-${UID}"
 
 # Route spec-decoding=mtp configs to the _mtp benchmark script (parity with
 # the h200 launchers, which have carried SPEC_SUFFIX since #392).
@@ -22,14 +27,48 @@ set -x
 JOB_ID=$(set +o pipefail; salloc --partition=$PARTITION --exclude=chi-mi300x-049,chi-mi300x-121 --gres=gpu:$GPU_COUNT --cpus-per-task=256 --time=180 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+')
 
 if [ -z "$JOB_ID" ]; then
-    echo "ERROR: salloc failed to allocate a job"
+    echo "ERROR: salloc failed to allocate a job" >&2
     exit 1
 fi
 
-trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT
+export PORT=$((40000 + (JOB_ID % 10000)))
+
+# The GitHub runners live on the Slurm controller, whose /home filesystem is
+# not mounted on compute nodes. Stage the checked-out source with sbcast, then
+# copy only benchmark artifacts back over srun stdout before releasing the job.
+LOCAL_WORKSPACE_TAR=$(mktemp "/tmp/inferencex-${JOB_ID}.XXXXXX.tar.gz")
+REMOTE_WORKSPACE_TAR="/tmp/inferencex-${JOB_ID}.tar.gz"
+REMOTE_WORKSPACE="/tmp/inferencex-${JOB_ID}"
+
+cleanup() {
+    local rc=$?
+    srun --jobid="$JOB_ID" bash -c "rm -rf '$REMOTE_WORKSPACE' '$REMOTE_WORKSPACE_TAR'" >/dev/null 2>&1 || true
+    scancel "$JOB_ID" 2>/dev/null || true
+    rm -f "$LOCAL_WORKSPACE_TAR"
+    exit "$rc"
+}
+trap cleanup EXIT
+
+tar -C "$GITHUB_WORKSPACE" \
+    --exclude=.git --exclude='*/.git' \
+    -czf "$LOCAL_WORKSPACE_TAR" .
+sbcast --jobid="$JOB_ID" --force "$LOCAL_WORKSPACE_TAR" "$REMOTE_WORKSPACE_TAR"
+srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c "
+    set -euo pipefail
+    mkdir -p '$XDG_RUNTIME_DIR'
+    chmod 700 '$XDG_RUNTIME_DIR'
+    mkdir -p '$COMPUTE_TMPDIR'
+    chmod 700 '$COMPUTE_TMPDIR'
+    rm -rf '$REMOTE_WORKSPACE'
+    mkdir -p '$REMOTE_WORKSPACE'
+    tar -C '$REMOTE_WORKSPACE' -xzf '$REMOTE_WORKSPACE_TAR'
+"
 
 # Use flock to serialize concurrent imports to the same squash file
-srun --jobid=$JOB_ID --job-name="$RUNNER_NAME" bash -c "
+srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c "
+    set -euo pipefail
+    export TMPDIR='$COMPUTE_TMPDIR'
+    mkdir -p '$SQUASH_DIR'
     exec 9>\"$LOCK_FILE\"
     flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; }
     if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then
@@ -39,14 +78,35 @@ srun --jobid=$JOB_ID --job-name="$RUNNER_NAME" bash -c "
         enroot import -o \"$SQUASH_FILE\" docker://$IMAGE
     fi
 "
-srun --jobid=$JOB_ID \
+
+set +e
+srun --jobid="$JOB_ID" \
 --container-image=$SQUASH_FILE \
---container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,/dev/kfd:/dev/kfd,/dev/dri:/dev/dri \
+--container-mounts=$REMOTE_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,/dev/kfd:/dev/kfd,/dev/dri:/dev/dri \
 --container-mount-home \
 --container-writable \
 --container-remap-root \
 --container-workdir=/workspace/ \
 --no-container-entrypoint --export=ALL \
 bash benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_mi300x${SPEC_SUFFIX}.sh
+BENCHMARK_RC=$?
+set -e
+
+# Stream runtime artifacts back to the controller-side Actions workspace. The
+# binary archive is stdout-only; Slurm diagnostics remain on stderr.
+srun --jobid="$JOB_ID" bash -c "
+    set -euo pipefail
+    cd '$REMOTE_WORKSPACE'
+    shopt -s nullglob
+    artifacts=()
+    for path in results LOGS *.json *.log gpu_metrics.csv profile_*.trace.json.gz eval_results*; do
+        [[ -e \"\$path\" ]] && artifacts+=(\"\$path\")
+    done
+    if (( \${#artifacts[@]} == 0 )); then
+        tar -czf - --files-from /dev/null
+    else
+        tar -czf - \"\${artifacts[@]}\"
+    fi
+" | tar -C "$GITHUB_WORKSPACE" -xzf -
 
-scancel $JOB_ID
+exit "$BENCHMARK_RC"