diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh new file mode 100755 index 000000000..aed904746 --- /dev/null +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -eo pipefail +set -x + +# Qwen3.5-397B-A17B FP8 on MI300X with native EAGLE/NEXTN MTP. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars \ + MODEL TP CONC EP_SIZE KV_OFFLOADING \ + RESULT_DIR DURATION + +require_agentic_kv_offload_none + +SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +rocm-smi || true +amd-smi || true + +export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k +resolve_trace_source +install_agentic_deps + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +PARALLEL_ARGS=( + --tensor-parallel-size "$TP" + --data-parallel-size 1 + --ep-size "$EP_SIZE" +) + +# AgentX concurrency counts live session trees, not individual HTTP requests. +MAX_RUNNING_REQUESTS=$((2 * CONC)) +CUDA_GRAPH_MAX_BS="$CONC" +[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 + +export PYTHONNOUSERSITE=1 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_TIMEOUT_KEEP_ALIVE=1800 + +if [ "${EVAL_ONLY:-false}" != "true" ]; then + export SGLANG_SIMULATE_ACC_LEN=3.39 + export SGLANG_SIMULATE_ACC_METHOD=match-expected + export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token +fi + +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --trust-remote-code + "${PARALLEL_ARGS[@]}" + --attention-backend aiter + --enable-aiter-allreduce-fusion + --mamba-ssm-dtype bfloat16 + --tokenizer-worker-num 6 + --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --max-running-requests "$MAX_RUNNING_REQUESTS" + --max-prefill-tokens 32768 + --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" + --mem-fraction-static 0.75 + --tokenizer-path "$MODEL" + --reasoning-parser qwen3 + --tool-call-parser qwen3_coder + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 + --enable-metrics + --enable-cache-report +) + +write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" +"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "${EVAL_ONLY:-false}" = "true" ]; then + run_eval --port "$PORT" +else + # Aggregate serving exposes one logical SGLang Prometheus target. + export AIPERF_SERVER_METRICS_URLS="http://localhost:$PORT/metrics" + export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 2ce535a42..b4b6b48cf 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -402,6 +402,23 @@ qwen3.5-fp8-mi300x-sglang: search-space: - { tp: 8, conc-start: 4, conc-end: 64 } +# AgentX Pareto sweep for Qwen3.5 FP8 on MI300X. The fast discovery run peaks +# at c24; c32 records the post-knee throughput and latency cliff. +qwen3.5-fp8-mi300x-sglang-agentic-mtp: + image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730 + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:mi300x-amds + precision: fp8 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [4, 16, 20, 24, 32] } + dsr1-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 model: deepseek-ai/DeepSeek-R1-0528 @@ -1647,5 +1664,5 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4], spec-decoding: mtp } + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b76d628d7..9aeaf25d9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5825,3 +5825,15 @@ - "Add the MI355X MiniMax-M3 FP4 vLLM AgentX MTP submission on vLLM v0.27.1, using the committed EAGLE3 synthetic acceptance length of 3.35 for throughput and real target verification for eval." - "Sweep TP2/TP4 GPU-resident configurations and TP4 LMCache MP DRAM-offload points around the capacity knee, with vLLM server metrics enabled." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2487 + +- config-keys: + - qwen3.5-fp8-mi300x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add Qwen3.5-397B-A17B FP8 AgentX on MI300X with SGLang native EAGLE MTP and golden synthetic acceptance length 3.39" + - "Use TP8/EP1 and TP8/EP8 Pareto sweeps at c4/c16/c20/c24/c32; fast discovery found peak throughput at c24 and a post-knee regression at c32" + - "Exclude HiCache because SGLang v0.5.16 rejects Qwen3.5's layer-first hybrid cache layout" + - "Collect and require SGLang Prometheus metrics from the aggregate engine endpoint" + - "Image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2563 diff --git a/runners/launch_mi300x-amds.sh b/runners/launch_mi300x-amds.sh index fdd03889a..96d92e23d 100644 --- a/runners/launch_mi300x-amds.sh +++ b/runners/launch_mi300x-amds.sh @@ -1,12 +1,17 @@ #!/usr/bin/env bash -set -eo pipefail +set -euo pipefail export HF_HUB_CACHE_MOUNT="/raid/hf-hub-cache/" -export PORT=8888 PARTITION="compute" -SQUASH_FILE="/home/gharunner/gharunners/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" +SQUASH_DIR="/raid/hf-hub-cache/runtime-cache/enroot" +SQUASH_FILE="$SQUASH_DIR/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" LOCK_FILE="${SQUASH_FILE}.lock" +COMPUTE_TMPDIR="$SQUASH_DIR/tmp-${UID}" +# Some MI300X compute images do not provision the login user's /run/user/$UID +# directory. Keep Enroot's runtime state node-local instead of inheriting the +# controller's XDG_RUNTIME_DIR, which is not valid on those compute nodes. +export XDG_RUNTIME_DIR="/tmp/enroot-runtime-${UID}" # Route spec-decoding=mtp configs to the _mtp benchmark script (parity with # the h200 launchers, which have carried SPEC_SUFFIX since #392). @@ -22,14 +27,48 @@ set -x JOB_ID=$(set +o pipefail; salloc --partition=$PARTITION --exclude=chi-mi300x-049,chi-mi300x-121 --gres=gpu:$GPU_COUNT --cpus-per-task=256 --time=180 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') if [ -z "$JOB_ID" ]; then - echo "ERROR: salloc failed to allocate a job" + echo "ERROR: salloc failed to allocate a job" >&2 exit 1 fi -trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT +export PORT=$((40000 + (JOB_ID % 10000))) + +# The GitHub runners live on the Slurm controller, whose /home filesystem is +# not mounted on compute nodes. Stage the checked-out source with sbcast, then +# copy only benchmark artifacts back over srun stdout before releasing the job. +LOCAL_WORKSPACE_TAR=$(mktemp "/tmp/inferencex-${JOB_ID}.XXXXXX.tar.gz") +REMOTE_WORKSPACE_TAR="/tmp/inferencex-${JOB_ID}.tar.gz" +REMOTE_WORKSPACE="/tmp/inferencex-${JOB_ID}" + +cleanup() { + local rc=$? + srun --jobid="$JOB_ID" bash -c "rm -rf '$REMOTE_WORKSPACE' '$REMOTE_WORKSPACE_TAR'" >/dev/null 2>&1 || true + scancel "$JOB_ID" 2>/dev/null || true + rm -f "$LOCAL_WORKSPACE_TAR" + exit "$rc" +} +trap cleanup EXIT + +tar -C "$GITHUB_WORKSPACE" \ + --exclude=.git --exclude='*/.git' \ + -czf "$LOCAL_WORKSPACE_TAR" . +sbcast --jobid="$JOB_ID" --force "$LOCAL_WORKSPACE_TAR" "$REMOTE_WORKSPACE_TAR" +srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c " + set -euo pipefail + mkdir -p '$XDG_RUNTIME_DIR' + chmod 700 '$XDG_RUNTIME_DIR' + mkdir -p '$COMPUTE_TMPDIR' + chmod 700 '$COMPUTE_TMPDIR' + rm -rf '$REMOTE_WORKSPACE' + mkdir -p '$REMOTE_WORKSPACE' + tar -C '$REMOTE_WORKSPACE' -xzf '$REMOTE_WORKSPACE_TAR' +" # Use flock to serialize concurrent imports to the same squash file -srun --jobid=$JOB_ID --job-name="$RUNNER_NAME" bash -c " +srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c " + set -euo pipefail + export TMPDIR='$COMPUTE_TMPDIR' + mkdir -p '$SQUASH_DIR' exec 9>\"$LOCK_FILE\" flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; } if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then @@ -39,14 +78,35 @@ srun --jobid=$JOB_ID --job-name="$RUNNER_NAME" bash -c " enroot import -o \"$SQUASH_FILE\" docker://$IMAGE fi " -srun --jobid=$JOB_ID \ + +set +e +srun --jobid="$JOB_ID" \ --container-image=$SQUASH_FILE \ ---container-mounts=$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,/dev/kfd:/dev/kfd,/dev/dri:/dev/dri \ +--container-mounts=$REMOTE_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,/dev/kfd:/dev/kfd,/dev/dri:/dev/dri \ --container-mount-home \ --container-writable \ --container-remap-root \ --container-workdir=/workspace/ \ --no-container-entrypoint --export=ALL \ bash benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_mi300x${SPEC_SUFFIX}.sh +BENCHMARK_RC=$? +set -e + +# Stream runtime artifacts back to the controller-side Actions workspace. The +# binary archive is stdout-only; Slurm diagnostics remain on stderr. +srun --jobid="$JOB_ID" bash -c " + set -euo pipefail + cd '$REMOTE_WORKSPACE' + shopt -s nullglob + artifacts=() + for path in results LOGS *.json *.log gpu_metrics.csv profile_*.trace.json.gz eval_results*; do + [[ -e \"\$path\" ]] && artifacts+=(\"\$path\") + done + if (( \${#artifacts[@]} == 0 )); then + tar -czf - --files-from /dev/null + else + tar -czf - \"\${artifacts[@]}\" + fi +" | tar -C "$GITHUB_WORKSPACE" -xzf - -scancel $JOB_ID +exit "$BENCHMARK_RC"