diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index 1b9311eda..c5f0a95e7 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -107,6 +107,87 @@ mkdir -p "$RESULT_DIR" SERVER_PID="" ROUTER_PID="" MOONCAKE_MASTER_PID="" +LMCACHE_PID="" + +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$ROUTER_PID" "vLLM router" + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" + stop_background_process_tree "$LMCACHE_PID" "LMCache server" + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# Mooncake does not publish a ROCm wheel. Build the pinned release once for +# each immutable OS/Python/ROCm tuple and reuse the shared cache on later jobs. +install_mooncake_rocm() { + local mooncake_tag="v0.3.11.post1" + local mooncake_src="/tmp/Mooncake-$mooncake_tag" + local mooncake_stage="/tmp/mooncake-stage-$mooncake_tag" + local build_jobs cache_root cache_key cache_archive cache_tmp + local engine_path os_version python_abi rocm_version + + build_jobs=$(nproc) + if ((build_jobs > 32)); then + build_jobs=32 + fi + os_version=$(. /etc/os-release && printf '%s-%s' "$ID" "$VERSION_ID") + python_abi=$(python3 -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")') + rocm_version=$(sed -n '1p' /opt/rocm/.info/version 2>/dev/null || true) + if [[ -z "$rocm_version" ]]; then + rocm_version=$(hipconfig --version) + fi + rocm_version=${rocm_version//[^[:alnum:]._-]/_} + cache_root="${HF_HUB_CACHE:?HF_HUB_CACHE must be set}/inferencex/mooncake" + cache_key="${mooncake_tag}-${os_version}-${python_abi}-${rocm_version}-$(uname -m)-hip" + cache_archive="$cache_root/$cache_key.tar.gz" + mkdir -p "$cache_root" + + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential cmake git libasio-dev libboost-dev libcurl4-openssl-dev \ + libgflags-dev libgoogle-glog-dev libibverbs-dev libjsoncpp-dev \ + libnuma-dev libpython3-dev libssl-dev libunwind-dev liburing-dev \ + libxxhash-dev libyaml-cpp-dev libzstd-dev ninja-build pybind11-dev + + exec 9>"$cache_archive.lock" + flock -w 1800 9 + if [[ -f "$cache_archive" ]] && ! tar -tzf "$cache_archive" >/dev/null 2>&1; then + rm -f "$cache_archive" + fi + if [[ ! -f "$cache_archive" ]]; then + echo "Building HIP Mooncake cache artifact: $cache_archive" + rm -rf "$mooncake_src" "$mooncake_stage" + git clone --depth 1 --branch "$mooncake_tag" --recurse-submodules \ + --shallow-submodules https://github.com/kvcache-ai/Mooncake.git "$mooncake_src" + cmake -S "$mooncake_src/extern/yalantinglibs" \ + -B "$mooncake_src/extern/yalantinglibs/build" \ + -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF -DBUILD_UNIT_TESTS=OFF + cmake --build "$mooncake_src/extern/yalantinglibs/build" -j "$build_jobs" + cmake --install "$mooncake_src/extern/yalantinglibs/build" + cmake -S "$mooncake_src" -B "$mooncake_src/build" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DUSE_CUDA=OFF -DUSE_HIP=ON \ + -DWITH_EP=OFF -DWITH_STORE=ON -DWITH_STORE_RUST=OFF \ + -DWITH_RUST_EXAMPLE=OFF -DBUILD_EXAMPLES=OFF -DBUILD_UNIT_TESTS=OFF + cmake --build "$mooncake_src/build" -j "$build_jobs" + mkdir -p "$mooncake_stage" + DESTDIR="$mooncake_stage" cmake --install "$mooncake_src/build" + cache_tmp=$(mktemp "$cache_root/$cache_key.tmp.XXXXXX") + tar -C "$mooncake_stage" -czf "$cache_tmp" . + mv -f "$cache_tmp" "$cache_archive" + else + echo "Using HIP Mooncake cache artifact: $cache_archive" + fi + tar -C / -xzf "$cache_archive" + engine_path=$(python3 -c 'import mooncake.engine; print(mooncake.engine.__file__)') + ldd "$engine_path" | grep -q 'libamdhip64.so' + exec 9>&- +} OFFLOAD_ARGS=() @@ -139,30 +220,13 @@ case "${KV_OFFLOAD_BACKEND:-}" in ;; mooncake) require_agentic_kv_offload_backend mooncake - # ---- Mooncake config ---------------------------------------------------------- # Embedded mode contributes one segment per GPU rank to a shared - # distributed store, so pre-divide the aggregate host-memory budget. + # distributed store, so pre-divide the generated aggregate budget. PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) - - #MOONCAKE_VERSION=0.3.11.post1 - #apt-get update && apt-get install -y libcurl4 libibverbs1 rdma-core librdmacm1 libnuma1 liburing2 - #agentic_pip_install --quiet --no-cache-dir --no-deps \ - # --force-reinstall "mooncake-transfer-engine-non-cuda==$MOONCAKE_VERSION" - - git clone https://github.com/kvcache-ai/Mooncake.git - cd Mooncake - bash dependencies.sh - mkdir build - cd build - cmake .. - make -j - sudo make install # optional, make it ready to be used by vLLM/SGLang - cd .. - cd .. - + if ! python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null 2>&1; then + install_mooncake_rocm + fi python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - export INFERENCEX_MOONCAKE_MAX_TRANSFER_BATCH_KEYS=32 - python3 "$(dirname "$0")/patch_vllm_mooncake_transfer_batches.py" MOONCAKE_MASTER_PORT=$((PORT + 12000)) MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" @@ -178,22 +242,18 @@ case "${KV_OFFLOAD_BACKEND:-}" in "enable_offload": false } EOF -# (srok) - #"protocol": "rdma", - #"device_name": "mlx5_0", - #"local_buffer_size": "4GB", + # Mooncake v0.3.11.post1 emits its transfer polling loop at VLOG(1). + # Keep normal INFO diagnostics while suppressing that unbounded hot-loop output. + export GLOG_v=0 export MOONCAKE_CONFIG_PATH export MC_ENABLE_DEST_DEVICE_AFFINITY=1 export PYTHONHASHSEED=0 export MC_SLICE_SIZE=1048576 - # (srok) - #export MC_WORKERS_PER_CTX=4 export MC_WORKERS_PER_CTX=8 MOONCAKE_EVICTION_HIGH_WATERMARK_RATIO=0.80 MOONCAKE_EVICTION_RATIO=0.10 MOONCAKE_KV_LEASE_TTL=60s - #MOONCAKE_KV_LEASE_TTL=3600s echo "Starting Mooncake master on port $MOONCAKE_MASTER_PORT..." mooncake_master --port "$MOONCAKE_MASTER_PORT" \ @@ -219,30 +279,6 @@ EOF lmcache) require_agentic_kv_offload_backend lmcache # ---- Lmcache config ---------------------------------------------------------- - LMCACHE_PID="" - - cleanup_lmcache_server() { - if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then - kill "$LMCACHE_PID" 2>/dev/null || true - wait "$LMCACHE_PID" 2>/dev/null || true - fi - } - - trap cleanup_lmcache_server EXIT - - cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$ROUTER_PID" "vLLM router" - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" - exit "$exit_code" - } - trap cleanup_agentic_services EXIT - trap 'exit 130' INT - trap 'exit 143' TERM - wait_for_lmcache_ready() { { set +x; } 2>/dev/null local attempts="${LMCACHE_READY_ATTEMPTS:-120}" @@ -379,6 +415,9 @@ MAX_NUM_SEQS=$((2 * CONC)) # DeepSeek-V4-Pro ships a native MTP head. AgentX throughput pins its # three-token draft to the committed thinking-on golden acceptance length; # eval-only runs use real target verification so accuracy remains meaningful. +# The isolated scheduler, MTP-depth, graph-mode, collective, and breakable- +# graph probes did not produce a clean improvement. Keep the verified +# K=3/8K/FULL_DECODE_ONLY baseline while expanding topology coverage. NUM_SPEC_TOKENS=3 SYNTHETIC_ACCEPT_LEN=2.49 if [ "${EVAL_ONLY:-false}" = "true" ]; then @@ -387,14 +426,21 @@ else SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" fi +# DEP4 replicates the attention weights across four DP ranks. At the common +# 0.90 setting vLLM leaves only 12.06--12.50 GiB for KV on MI355X, below the +# 14.88 GiB required to admit one native 1,048,576-token request. Give only +# this topology the extra headroom it needs; established TP/TEP/DEP8 results +# retain their original memory fraction. +GPU_MEM_UTIL=0.9 +if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 4 ]; then + GPU_MEM_UTIL=0.92 +fi + echo "Starting vllm server..." set -x export VLLM_ROCM_USE_AITER=1 -#export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 export VLLM_ROCM_USE_AITER_MOE=1 -sleep 180 - { set +x; } 2>/dev/null VLLM_CMD=( vllm serve "$MODEL_PATH" --served-model-name "$MODEL" @@ -406,15 +452,18 @@ VLLM_CMD=( --kv-cache-dtype fp8 "${PARALLEL_ARGS[@]}" "${EP_ARGS[@]}" - --gpu-memory-utilization 0.8 + --gpu-memory-utilization "$GPU_MEM_UTIL" + --block-size 256 + --max-num-batched-tokens 8192 --moe-backend aiter - --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' + --compilation-config '{"mode":3,"cudagraph_mode":"FULL_DECODE_ONLY"}' --speculative-config "$SPEC_CONFIG" --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 --reasoning-parser deepseek_v4 --enable-auto-tool-choice --enable-prefix-caching + --enable-prompt-tokens-details --no-disable-hybrid-kv-cache-manager --max-num-seqs "$MAX_NUM_SEQS" "${OFFLOAD_ARGS[@]}" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d26796a52..e7b04e349 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1227,13 +1227,13 @@ qwen3.5-fp8-mi355x-sglang-agentic-hicache: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48, 64] } -# MTP twin of dsv4-fp4-mi355x-vllm-agentic. The topologies, KV-offload backends, -# and concurrency points are identical; speculative decoding is enabled on the -# current immutable ROCm nightly because the base config's retired nightly tag -# is no longer available from Docker Hub. Throughput uses the three-token golden -# synthetic acceptance length, while eval-only runs retain real verification. +# DeepSeek-V4-Pro FP4 MTP collection sweep for MI355X. Keep the verified +# GPU-resident frontier, add denser c52/c60/c68 TEP guards, collect the same +# points with pinned Mooncake DRAM offload, and retain representative DEP8 and +# DEP4 cells even when dominated. The fast pass is a functionality gate; +# Pareto optimality is deliberately not required for inclusion. dsv4-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-821717118fc26667dd474b9b0ab81d29259dfc5c + image: vllm/vllm-openai-rocm:v0.27.0 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1244,11 +1244,17 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.60 search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 40, 48] } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64], router: { name: vllm-router, version: "0.1.14" } } - # LMCache invalid-block recovery currently assumes one KV-cache group, - # while MTP creates two. Restore these points after the upstream hybrid - # KV recovery fix lands: https://github.com/vllm-project/vllm/pull/45497 + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8, 12] } + - { tp: 8, ep: 8, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [16, 24, 32, 40, 48, 52, 56, 60, 64, 68] } + - { tp: 8, ep: 8, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, spec-decoding: mtp, conc-list: [52, 60, 68] } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128, 192, 224], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8], router: { name: vllm-router, version: "0.1.14" } } + # DEP4 previously had only 12.09 GiB/GPU available for the native + # 1,048,576-token context versus a 14.88 GiB minimum. Retain two cells in + # the fast gate to reconfirm whether the current base can now initialize; + # do not weaken the AgentX context limit to make them pass. + # The stable-v0.27 Mooncake c52 gate completed its full 575-request + # warmup and measured profile without hybrid-KV or request errors. dsr1-fp4-mi355x-sglang-disagg-mtp: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 64f87b447..ea1562333 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5726,6 +5726,17 @@ - "Use supported header-based Dynamo session routing with the in-repo AIPerf build." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2520 +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh the MI355X DeepSeek-V4-Pro MTP submission on stable vLLM ROCm v0.27.0; raise GPU-memory utilization from 0.80 to 0.90 and use 256-token blocks, an 8192-token scheduler budget, and FULL_DECODE_ONLY graphs." + - "Keep the measured MTP operating point at three draft tokens with golden synthetic acceptance length 2.49." + - "Expand collection to TP8 c1-c12, TP8/EP8 c16-c68, Mooncake DRAM offload c52/c60/c68, DEP8 c64-c224, and DEP4 c4/c8, including dominated points requested for comparison." + - "Use 0.92 GPU-memory utilization only for DEP4 so the native 1,048,576-token context fits; all other points retain 0.90." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2540 + - config-keys: - dsv4-fp4-gb300-dynamo-sglang-agentic-agg - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg