diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh new file mode 100755 index 000000000..3518a85a7 --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# MiniMax-M3 MXFP8 H100 AgentX (agentic-coding) recipe with EAGLE3 speculative +# decoding — the spec-decoding=mtp variant of agentic/minimaxm3_fp8_h100.sh. +# Everything outside the speculative block mirrors the non-MTP agentic sibling +# (Mooncake host-DRAM KV offload, --block-size 128, --language-model-only, +# --kv-cache-dtype fp8, TRITON_ATTN, minimax_m3 parsers, vllm-router for +# DP-attention), so the spec-decode delta is readable at equal concurrency. +# +# Speculative config: the current Inferact/MiniMax-M3-EAGLE3-GQA draft head +# with three speculative tokens and the committed thinking-on golden AL. +# +# The drafter is pinned to FLASH_ATTN, as on every CUDA MiniMax-M3 MTP recipe: +# the EAGLE3 head is MHA and FlashInfer only serves page size 128 through its +# trtllm-gen kernel, which requires GQA/MQA. FLASH_ATTN accepts any +# multiple-of-16 block size, so the mandatory 128 is fine for the draft. (The +# ROCm recipes need no pin because their server runs TRITON_ATTN throughout.) +# +# Throughput runs pin synthetic acceptance to the committed golden AL; the +# EVAL_ONLY accuracy run keeps real target verification. See SYNTHETIC_ACCEPT_LEN. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +# The EAGLE3 draft is never pre-staged next to the target checkpoint; fetch it +# into the shared HF cache. That cache is a network FS where concurrent +# day-zero downloads hit huggingface_hub's WeakFileLock "[Errno 116] Stale file +# handle" race, so retry (the download resumes) as the fixed-seq-len MTP +# recipes do. +for attempt in 1 2 3 4 5; do + hf download "$DRAFT_MODEL" && break + if [ "$attempt" = 5 ]; then echo "hf download of $DRAFT_MODEL failed after $attempt attempts" >&2; exit 1; fi + echo "hf download attempt $attempt failed; retrying in 60s" >&2 + sleep 60 +done +nvidia-smi + +export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 +resolve_trace_source +install_agentic_deps + +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export PYTHONNOUSERSITE=1 + +SERVER_LOG="$RESULT_DIR/server.log" +ROUTER_LOG="$RESULT_DIR/router.log" +MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +mkdir -p "$RESULT_DIR" + +OFFLOAD_ARGS=() +MODEL_CPU_OFFLOAD_GB=26 +MODEL_CHECKPOINT_PAGE_CACHE_GIB=414 +MOONCAKE_LOCAL_BUFFER_GIB=4 +case "${KV_OFFLOAD_BACKEND:-}" in + "") + require_agentic_kv_offload_none + ;; + mooncake) + require_agentic_kv_offload_backend mooncake + TOTAL_CPU_DRAM_GIB=$((TOTAL_CPU_DRAM_GB * 1000000000 / 1073741824)) + PER_RANK_GIB=$(((TOTAL_CPU_DRAM_GIB - MODEL_CHECKPOINT_PAGE_CACHE_GIB) / TP - MODEL_CPU_OFFLOAD_GB - MOONCAKE_LOCAL_BUFFER_GIB)) + if (( PER_RANK_GIB <= 0 )); then + echo "Error: CPU DRAM budget is too small for checkpoint cache, model, and KV offload" >&2 + exit 1 + fi + MOONCAKE_VERSION=0.3.11.post1 + agentic_pip_install --quiet --no-cache-dir --no-deps \ + --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" + python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + MOONCAKE_MASTER_PORT=$((PORT + 12000)) + MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" + cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & + MOONCAKE_MASTER_PID=$! + sleep 2 + kill -0 "$MOONCAKE_MASTER_PID" + OFFLOAD_ARGS=( + --kv-transfer-config + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2 + exit 1 + ;; +esac + +PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) +if [[ "$DP_ATTENTION" == "true" ]]; then + PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") +fi + +EP_ARGS=() +if (( EP_SIZE > 1 )); then + EP_ARGS=(--enable-expert-parallel) +fi + +VLLM_BACKEND_PORT="$PORT" +if [[ "$DP_ATTENTION" == "true" ]]; then + VLLM_BACKEND_PORT=$((PORT + 1)) + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 + agentic_pip_install --quiet 'vllm-router==0.1.14' +fi + +# The public endpoint is the router for DEP, so explicitly scrape the engine +# endpoint. Pure TP/TEP deduplicates this URL against the automatic scrape. +export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" + +# use 3 speculative tokens for all configs, matching the MiniMax-M3 MTP recipes +NUM_SPEC_TOKENS=3 +TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) + +# AgentX pins acceptance to the committed golden AL so submissions are compared +# on system performance at a fixed acceptance target rather than on draft-head +# quality. 2.78 is minimaxm3_eagle3_gqa.yaml thinking_on[3]. +# +# EVAL_ONLY switches back to real verification: synthetic acceptance commits +# drafted tokens regardless of the target logits, so generated text is wrong and +# the eval would score ~0 (same split as dsv4_fp4_b*_vllm_mtp.sh). +SYNTHETIC_ACCEPT_LEN=2.78 +if [ "${EVAL_ONLY:-false}" = "true" ]; then + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" +else + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" +fi + +# AgentX concurrency counts live session trees, not individual requests. DEP +# splits those trees across eight data-parallel ranks; TP/TEP keeps them local. +if [[ "$DP_ATTENTION" == "true" ]]; then + if (( 2 * CONC % TP != 0 )); then + echo "DEP requires 2*CONC divisible by TP (CONC=$CONC TP=$TP)" >&2 + exit 1 + fi + MAX_NUM_SEQS=$((2 * CONC / TP)) +else + MAX_NUM_SEQS=$((2 * CONC)) +fi +# Cudagraph capture sizes are in TOKENS: a decode batch of S sequences verifies +# S*(1+NUM_SPEC_TOKENS) tokens, so cap capture at MAX_NUM_SEQS*(1+N) or the +# FULL_DECODE_ONLY ladder tops out at MAX_NUM_SEQS/(1+N) sequences and the +# largest decode batches fall back to eager. +MAX_CUDAGRAPH_CAPTURE_SIZE=$((MAX_NUM_SEQS * TOKENS_PER_SEQ)) + +vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ + --host 0.0.0.0 \ + --port "$VLLM_BACKEND_PORT" \ + "${PARALLEL_ARGS[@]}" \ + "${EP_ARGS[@]}" \ + --gpu-memory-utilization 0.90 \ + --cpu-offload-gb "$MODEL_CPU_OFFLOAD_GB" \ + --kv-cache-dtype fp8 \ + --attention-backend TRITON_ATTN \ + --block-size 128 \ + --language-model-only \ + --enable-prefix-caching \ + --enable-prompt-tokens-details \ + --default-chat-template-kwargs '{"thinking_mode":"enabled"}' \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --max-cudagraph-capture-size "$MAX_CUDAGRAPH_CAPTURE_SIZE" \ + --speculative-config "$SPEC_CONFIG" \ + --tool-call-parser minimax_m3 \ + --reasoning-parser minimax_m3 \ + --enable-auto-tool-choice \ + --safetensors-load-strategy lazy \ + --trust-remote-code \ + "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [[ "$DP_ATTENTION" == "true" ]]; then + vllm-router \ + --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ + --policy consistent_hash \ + --intra-node-data-parallel-size "$TP" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$((PORT + 10000))" \ + --request-timeout-secs 14400 \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c752b04f6..51c853352 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7141,6 +7141,35 @@ minimaxm3-fp8-h100-vllm-agentic: - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } +# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of +# minimaxm3-fp8-h100-vllm-agentic, pairing MiniMaxAI/MiniMax-M3-MXFP8 with the +# Inferact/MiniMax-M3-EAGLE3-GQA draft head (3 speculative tokens, FLASH_ATTN +# drafter) and pinning synthetic acceptance to the golden AL 2.78 +# (golden_al_distribution/minimaxm3_eagle3_gqa.yaml, thinking_on, K=3). Same TP8-only +# layout and KV arms as the non-MTP entry so the spec-decode delta is readable at +# equal concurrency, trimmed at the extreme-conc end: the draft head plus its KV +# eat into the same HBM budget that already puts the GPU-resident cliff near +# conc 6 on 80 GB H100s. +minimaxm3-fp8-h100-vllm-agentic-mtp: + image: vllm/vllm-openai:v0.27.1 + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: cluster:h100-dgxc + precision: fp8 + framework: vllm + multinode: false + scenarios: + agentic-coding: + # The complete fast sweep places the GPU-resident cliff between c5 and c6. + # Retain the resident latency/knee curve through c5, then use Mooncake at + # c6/c8 to extend throughput without crossing that HBM cliff. TEP was + # dominated, DEP could not allocate its 1M-token KV cache on 80 GB H100s, + # and the vLLM-simple function screen did not justify a frontier row. + - dram-utilization: 0.80 + search-space: + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [6, 8] } + minimaxm3-fp8-h200-vllm-agentic: image: vllm/vllm-openai:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 model: MiniMaxAI/MiniMax-M3-MXFP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0bce8d894..2484290a9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5818,3 +5818,13 @@ description: - "Extend the SimpleCPUOffloadConnector grid to c8/c12/c16/c20/c24/c28/c32/c48/c64 to locate its crossover against the resident curve" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 + +- config-keys: + - minimaxm3-fp8-h100-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Refresh H100 MiniMax-M3 MXFP8 AgentX with vLLM v0.27.1 and EAGLE3-GQA synthetic golden AL 2.78." + - "Use the complete fast sweep to retain TP8 c1-c5 and Mooncake TP8 c6/c8 for the strict Pareto sweep; TEP is dominated, DEP is infeasible at its required KV allocation, and vLLM-simple is not retained after its function screen." + - "Require vLLM server metrics and keep strict request and profile validation." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2564