diff --git a/benchmarks/single_node/agentic/dsv4_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp8_mi325x_mtp.sh new file mode 100755 index 000000000..f2e1617b3 --- /dev/null +++ b/benchmarks/single_node/agentic/dsv4_fp8_mi325x_mtp.sh @@ -0,0 +1,308 @@ +#!/usr/bin/env bash +set -euo pipefail + +# DeepSeek-V4-Pro FP8 AgentX replay on one 8xMI325X node. The checkpoint is +# dequantized to FP8 because gfx942 has no native MXFP4 support. The collection +# includes pure TP8, TP8/EP8, and Mooncake DRAM-offload points explicitly, even +# where the latter two do not improve the Pareto frontier. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL IMAGE TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +resolve_trace_source +install_agentic_deps +agentic_pip_install --quiet Pillow fastapi uvicorn + +install_vllm_hybrid_kv_recovery() { + # vLLM PR #45497 fixes hybrid-KV connector load failures by recomputing the + # complete request when any KV group is unavailable. Keep v0.27.0's ROCm + # binaries and install only the pinned Python scheduler source revision. + local recovery_commit="cb1917efe34f16423de2ceb7f210fd015d53459a" + local recovery_src="/tmp/vllm-hybrid-kv-recovery" + rm -rf "$recovery_src" + git clone --depth 1 --branch test/v027-hybrid-kv-recovery \ + https://github.com/cquil11/vllm.git "$recovery_src" + if [[ "$(git -C "$recovery_src" rev-parse HEAD)" != "$recovery_commit" ]]; then + echo "Unexpected vLLM hybrid-KV recovery revision" >&2 + exit 1 + fi + VLLM_USE_PRECOMPILED=1 \ + VLLM_ROCM_WHEEL_INDEX=https://wheels.vllm.ai/rocm/0.27.0/rocm723 \ + uv pip install --system --no-deps --no-build-isolation --editable \ + "$recovery_src" + python3 -c 'import vllm; print("vLLM source:", vllm.__file__)' +} + +export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 +export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" +export VLLM_ENGINE_READY_TIMEOUT_S=10800 +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_USE_AITER_MOE=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export PYTHONNOUSERSITE=1 + +SERVER_LOG="$RESULT_DIR/server.log" +ROUTER_LOG="$RESULT_DIR/router.log" +MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +ROUTER_PID="" +MOONCAKE_MASTER_PID="" + +# Mooncake does not publish a ROCm wheel. Reuse the established MI300X/MI325X +# recipe convention: build the pinned release once per immutable ROCm/Python +# combination, cache the staged install on shared storage, and verify that the +# loaded transfer engine is actually linked against HIP. +install_mooncake_rocm() { + local mooncake_tag="v0.3.11.post1" + local mooncake_src="/tmp/Mooncake-$mooncake_tag" + local mooncake_stage="/tmp/mooncake-stage-$mooncake_tag" + local build_jobs cache_root cache_key cache_archive cache_tmp + local engine_path os_version python_abi rocm_version + + build_jobs=$(nproc) + if ((build_jobs > 32)); then + build_jobs=32 + fi + + os_version=$(. /etc/os-release && printf '%s-%s' "$ID" "$VERSION_ID") + python_abi=$(python3 -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")') + rocm_version=$(sed -n '1p' /opt/rocm/.info/version 2>/dev/null || true) + if [[ -z "$rocm_version" ]]; then + rocm_version=$(hipconfig --version) + fi + rocm_version=${rocm_version//[^[:alnum:]._-]/_} + cache_root="${HF_HUB_CACHE:?HF_HUB_CACHE must be set}/inferencex/mooncake" + cache_key="${mooncake_tag}-${os_version}-${python_abi}-${rocm_version}-$(uname -m)-hip" + cache_archive="$cache_root/$cache_key.tar.gz" + mkdir -p "$cache_root" + + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential cmake git libasio-dev libboost-dev libcurl4-openssl-dev \ + libgflags-dev libgoogle-glog-dev libibverbs-dev libjsoncpp-dev \ + libnuma-dev libpython3-dev libssl-dev libunwind-dev liburing-dev \ + libxxhash-dev libyaml-cpp-dev libzstd-dev ninja-build pybind11-dev + + exec 9>"$cache_archive.lock" + flock -w 1800 9 + if [[ -f "$cache_archive" ]] && ! tar -tzf "$cache_archive" >/dev/null 2>&1; then + rm -f "$cache_archive" + fi + if [[ ! -f "$cache_archive" ]]; then + echo "Building HIP Mooncake cache artifact: $cache_archive" + rm -rf "$mooncake_src" "$mooncake_stage" + git clone --depth 1 --branch "$mooncake_tag" --recurse-submodules \ + --shallow-submodules https://github.com/kvcache-ai/Mooncake.git "$mooncake_src" + cmake -S "$mooncake_src/extern/yalantinglibs" \ + -B "$mooncake_src/extern/yalantinglibs/build" \ + -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF -DBUILD_UNIT_TESTS=OFF + cmake --build "$mooncake_src/extern/yalantinglibs/build" -j "$build_jobs" + cmake --install "$mooncake_src/extern/yalantinglibs/build" + cmake -S "$mooncake_src" -B "$mooncake_src/build" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DUSE_CUDA=OFF -DUSE_HIP=ON \ + -DWITH_EP=OFF -DWITH_STORE=ON -DWITH_STORE_RUST=OFF \ + -DWITH_RUST_EXAMPLE=OFF -DBUILD_EXAMPLES=OFF -DBUILD_UNIT_TESTS=OFF + cmake --build "$mooncake_src/build" -j "$build_jobs" + mkdir -p "$mooncake_stage" + DESTDIR="$mooncake_stage" cmake --install "$mooncake_src/build" + cache_tmp=$(mktemp "$cache_root/$cache_key.tmp.XXXXXX") + tar -C "$mooncake_stage" -czf "$cache_tmp" . + mv -f "$cache_tmp" "$cache_archive" + else + echo "Using HIP Mooncake cache artifact: $cache_archive" + fi + tar -C / -xzf "$cache_archive" + engine_path=$(python3 -c 'import mooncake.engine; print(mooncake.engine.__file__)') + ldd "$engine_path" | grep -q 'libamdhip64.so' + exec 9>&- +} + +OFFLOAD_ARGS=() +if agentic_kv_offload_enabled; then + require_agentic_kv_offload_backend mooncake + install_vllm_hybrid_kv_recovery + # TOTAL_CPU_DRAM_GB is the generator-capped aggregate node budget. + # Embedded Mooncake contributes one segment per rank, so divide it here. + PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) + if ! python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null 2>&1; then + install_mooncake_rocm + fi + python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + MOONCAKE_MASTER_PORT=$((PORT + 12000)) + MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" + cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & + MOONCAKE_MASTER_PID=$! + sleep 2 + kill -0 "$MOONCAKE_MASTER_PID" + OFFLOAD_ARGS=( + --kv-transfer-config + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true}}' + ) +else + require_agentic_kv_offload_none +fi + +PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) +if [[ "$DP_ATTENTION" == "true" ]]; then + PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") +fi + +EP_ARGS=() +if (( EP_SIZE > 1 )); then + EP_ARGS=(--enable-expert-parallel) +fi + +USE_VLLM_ROUTER=false +VLLM_BACKEND_PORT="$PORT" +if [[ "$DP_ATTENTION" == "true" ]]; then + if (( EP_SIZE != TP )); then + echo "ERROR: MI325X DP-attention requires EP_SIZE == TP so FP8 experts remain sharded" >&2 + exit 1 + fi + USE_VLLM_ROUTER=true + VLLM_BACKEND_PORT=$((PORT + 1)) + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 + export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics" + agentic_pip_install --quiet 'vllm-router==0.1.14' +fi + +# The 16K prefill budget and 4*CONC sequence-cap probes were neutral, while +# piecewise graphs regressed. Restore the official 8K/2*CONC/FULL_DECODE_ONLY +# baseline before isolating INT4 Quick Reduce. +MAX_NUM_SEQS=$((2 * CONC)) +# The existing MI300X/MI325X fixed-sequence recipes use K=2. Keep that MTP +# depth and its measured golden acceptance length for every AgentX point. +NUM_SPEC_TOKENS=2 +SYNTHETIC_ACCEPT_LEN=2.27 +if [[ "${EVAL_ONLY:-false}" == "true" ]]; then + SPEC_CONFIG="{\"method\":\"mtp\",\"num_speculative_tokens\":${NUM_SPEC_TOKENS}}" +else + SPEC_CONFIG="{\"method\":\"mtp\",\"num_speculative_tokens\":${NUM_SPEC_TOKENS},\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":${SYNTHETIC_ACCEPT_LEN}}" +fi + +cleanup() { + local rc=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$ROUTER_PID" "vLLM router" + stop_background_process_tree "${SERVER_PID:-}" "vLLM server" 60 + stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$VLLM_BACKEND_PORT" + --trust-remote-code + --async-scheduling + --distributed-executor-backend mp + --quantization deepseek_v4_fp8 + --kv-cache-dtype fp8 + "${PARALLEL_ARGS[@]}" + "${EP_ARGS[@]}" + --gpu-memory-utilization 0.9 + --block-size 256 + --max-num-batched-tokens 8192 + --max-num-seqs "$MAX_NUM_SEQS" + --compilation-config '{"mode":3,"cudagraph_mode":"FULL_DECODE_ONLY"}' + --speculative-config "$SPEC_CONFIG" + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --reasoning-parser deepseek_v4 + --enable-auto-tool-choice + --enable-prefix-caching + --enable-prompt-tokens-details + --no-disable-hybrid-kv-cache-manager + "${OFFLOAD_ARGS[@]}" +) + +printf '%q ' "${VLLM_CMD[@]}" > "$RESULT_DIR/vllm_command.txt" +printf '\n' >> "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [[ "$USE_VLLM_ROUTER" == "true" ]]; then + vllm-router \ + --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ + --policy consistent_hash \ + --intra-node-data-parallel-size "$TP" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$((PORT + 10000))" \ + --request-timeout-secs 14400 \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +if [[ "${EVAL_ONLY:-false}" == "true" ]]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + # Full-context AgentX responses can remain healthy for several minutes + # after the admission window closes. Let already-admitted requests drain + # so their observed TTFT/ITL enters the strict coverage calculation. This + # does not extend admissions, change the workload, or lower the 98% gate. + REPLAY_CMD+=" --benchmark-grace-period 1800" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d26796a52..28d818e0d 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1471,6 +1471,29 @@ glm5.2-fp8-mi325x-sglang-agentic-mtp: search-space: - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5, 6, 8] } +# DeepSeek-V4-Pro FP8 full-context AgentX frontier on one MI325X node. The +# dequantized checkpoint is roughly 1.05 TB, so TP8 fits comfortably in the +# node's 2 TB of HBM while TP4 does not have enough per-rank model headroom. +# Stable v0.26.0 and v0.27.0 both produced invalid empty-content responses with +# expert parallelism, including strict K=1 and K=2 gates. Publish only the +# functional TP8 and Mooncake points. Mooncake uses the generated aggregate +# host-DRAM cap, the pinned ROCm build convention, and the upstream hybrid-KV +# load-recovery scheduler fix validated against v0.27.0. +dsv4-fp8-mi325x-vllm-agentic-mtp: + image: vllm/vllm-openai-rocm:v0.27.0 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi325x-amds + precision: fp8 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.90 + search-space: + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5, 6] } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, spec-decoding: mtp, conc-list: [4, 5, 6] } + minimaxm3-fp8-mi325x-vllm-agentic: image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 model: MiniMaxAI/MiniMax-M3-MXFP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 64f87b447..053ba4aba 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5726,6 +5726,35 @@ - "Use supported header-based Dynamo session routing with the in-repo AIPerf build." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2520 +- config-keys: + - dsv4-fp8-mi325x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the first DeepSeek-V4-Pro FP8 vLLM AgentX submission on MI325X, with MTP enabled for every published point; use two draft tokens and the committed K=2 golden synthetic acceptance length of 2.27, while eval-only runs retain real target verification." + - "Use one 8xMI325X node: the roughly 1.05 TB FP8-dequantized checkpoint fits in the node's 2 TB HBM at TP8, so multi-node execution would add communication without solving a capacity constraint; TP4 is excluded because its per-rank model footprint is not fit-safe." + - "The first broad agentx-fast pass covered dense TP8, TP8+EP8 and DP8+EP8 bands through concurrency 80. It found clean-server duration-coverage failures on slow TP points, repeated empty-content semantic responses on TP8+EP8 and DP8+EP8, and a DP c40 warmup request that did not drain within 1800 seconds." + - "Run a second seven-point agentx-fast diagnostic on stable vLLM v0.26.0 at TP8 c8/c12/c16, TP8+EP8 c20/c32, and DP8+EP8 c24/c48. This changes only MTP depth from K=3 to K=2 (and golden AL 2.49 to 2.27) so speculative depth is isolated before considering a nightly image or publishing the final dense frontier." + - "The K=2 stable pass reproduced the defect: TEP c20 had 32 empty-content errors in 39 warmup records, TEP c32 had 45 in 57, and DPA c24 aborted after five profiling semantic errors; TP c8 remained semantically clean but produced only 2,993 input and 10.5 output tokens/s. This isolates stable-v0.26 deepseek_v4_fp8 expert parallelism rather than MTP depth." + - "The immutable ROCm nightly nightly-f8d03e77416bf90c49acbe50e233275722f02c4b did not make expert parallelism publishable: TEP c20 produced 33 empty-content errors in 41 warmup records and 7 in its first 13 profile records, TEP c32 produced 51 in 63 warmup records, and DPA c24 produced 3 in 49 warmup records plus 1 in its first 11 profile records. Pure TP remained semantically clean." + - "Upstream's DeepSeek-V4-Pro recipe explicitly marks MI325X unsupported, and cross-node TP16 FP8 support remains an open, unmerged vLLM PR. Do not pin an unreleased patch: use the current immutable nightly and densely screen the fit-safe single-node pure-TP8 band at concurrency 1, 2, 4, 6, and 8 before the reusable full-duration sweep." + - "The pure-TP fast pass places the useful band at c1-c4: c1/c2 complete green at 13.49k/31.19 and 7.14k/44.53 input/output tok/s; c4 records 12.02k/67.97 with 0.67s p50 TTFT. c6/c8 collapse to 3.25-4.04k input and 9.70-12.60 output tok/s with 57-85s p50 TTFT, so drop that post-cliff region and probe the missing c3/c5 neighbors around c4." + - "Give already-admitted full-context responses up to 1800 seconds to drain after the measurement window, matching the merged MI325X AgentX convention. The c4 server remained healthy at roughly 84 output tok/s through the entire default 30-second grace, but two long responses were cancelled and completed-record TTFT coverage landed at 97.9% versus the unchanged 98% gate. Bounded drain preserves the admission window and strict coverage threshold." + - "The refined c3/c4/c5 fast pass completed all admitted requests with zero request errors and nonempty vllm: metrics exports. c3 reached 6.43k input / 64.35 output tok/s, c4 reached 11.15k / 69.11 with 0.68s p50 TTFT, and c5 fell to 6.30k / 33.04 with 7.8s p50 TTFT. Select c1-c4 as the reusable full-duration candidate grid and drop the clearly dominated c5-c8 region." + - "The reusable full-duration sweep invalidated c4 as a publishable point despite a healthy server and 295/295 successful requests: the finite five-trajectory corpus had only three distinct root session trees active at the 98% cutoff for configured concurrency four, so TTFT coverage correctly landed at 97.9504% while ITL remained 100%. Keep the strict gate, do not waive or shorten the workload, and publish the fully covered c1-c3 grid instead." + - "After the c1-c3 exact-head sweep and c3 eval completed fully green, run one isolated agentx-fast scheduler-budget comparison: increase only --max-num-batched-tokens from the official 8192-token ROCm baseline to 16384 while retaining the exact same c1/c2/c3 topology, image, MTP depth, acceptance length, KV dtype, graph mode, and workload. The completed 8192-token run remains the rollback point." + - "The isolated 16384-token c1 fast probe completed 48/48 requests with zero errors, TTFT 99.7%, ITL 100%, and 13,488.30 input / 31.19 output tok/s: exactly neutral versus the retained 8192-token result. Its c3 eval also completed 1,319/1,319 HTTP 200 responses. Reject the larger budget and restore 8192." + - "Next isolate only scheduler slot capacity at c3: raise --max-num-seqs from 2*CONC to 4*CONC while keeping the restored 8192-token budget and every topology, model, MTP, KV, graph, memory, and workload setting fixed. Baseline logs reached the 2*CONC running-request ceiling with queued branch work while KV use stayed below 18%, making c3 the informative upper retained point." + - "Reject the 4*CONC sequence cap after its c3 fast run completed fully green at 6,440.50 input / 64.42 output tok/s over 1,392.11 seconds, versus 6,433.20 / 64.35 over 1,393.73 seconds for the matched 2*CONC control: both token rates changed by only 0.11%. Its c3 eval also completed 1,319/1,319 HTTP 200 responses and collect-evals passed. Restore 2*CONC before any further experiment." + - "Next isolate only graph capture mode at c3: switch FULL_DECODE_ONLY to FULL_AND_PIECEWISE while keeping K=2/golden AL 2.27, the official 8192-token scheduler budget, max-num-seqs=2*CONC, FP8 KV, 90% GPU-memory utilization, block size 256, topology, image, and AgentX trace unchanged. Continue requiring nonempty vllm: server metrics from the explicit AIPerf endpoint." + - "Reject FULL_AND_PIECEWISE after the exact c3 fast comparison processed the same 76 requests with zero errors but reached 6,358.88 input / 63.60 output tok/s over 1,410.00 seconds, versus 6,433.20 / 64.35 over 1,393.73 seconds for the duration-matched FULL_DECODE_ONLY control: both rates regressed about 1.16%. TTFT coverage was 99.8%, ITL coverage was 100%, and server metrics exports were nonempty, so the longer completion time is a measured graph-mode regression rather than infrastructure noise." + - "Next isolate only INT4 Quick Reduce at the retained c3 baseline: enable VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 while restoring FULL_DECODE_ONLY and keeping the immutable nightly, TP8, K=2/golden AL 2.27, the 8192-token scheduler budget, max-num-seqs=2*CONC, FP8 KV, 90% GPU-memory utilization, block size 256, and the exact AgentX trace unchanged. The paired eval guards accuracy." + - "Expand the collection grid beyond the measured Pareto candidates: retain pure TP8 c1-c6 and add Mooncake DRAM-offload TP8 c4/c5/c6 using the generated aggregate host-memory cap and pinned ROCm Mooncake 0.3.11.post1 build." + - "Omit TEP8 c4-c6 after strict v0.27.0 gates reproduced invalid empty-content responses with both real MTP K=2 (3/10 profiling requests failed) and K=1 (2/18 failed). Keep the standard 10% request-error gate with no topology-specific exception." + - "Set Mooncake's independent client-heartbeat TTL to 120 seconds for the full-context offload points. The first c4-c6 run kept the 120-second object lease but used Mooncake's 10-second client default; a healthy rank was unmounted after missing that heartbeat window. A full-primer c4 isolation then kept all eight clients registered but exposed a second issue: the 0.3.11 TCP connection pool retained about 7,000 simultaneous sockets and 60-second batch transfers timed out. Match the proven MI355X convention by disabling that pool, enabling destination affinity, and using eight transfer workers." + - "Use the stable vLLM v0.27.0 ROCm image. For Mooncake points only, pin the Python-only scheduler recovery from upstream vLLM PR #45497 and set kv_load_failure_policy=recompute; the isolated c6 gate completed 65/65 warmup and 31/31 measured requests with zero errors while safely recomputing 22 partial hybrid-KV misses." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2541 + - config-keys: - dsv4-fp4-gb300-dynamo-sglang-agentic-agg - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg