diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml new file mode 100644 index 0000000000..8b02f5e857 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml @@ -0,0 +1,153 @@ +name: "disagg-h200-2p2d-pcp8-tp8-dp8-mtp" + +# Refresh of the 2P2D topology from Actions run 30133535261 with EAGLE MTP. +# The released SGLang image does not combine MTP with HiSparse, so decode KV +# cache remains GPU-resident. MTP uses the committed GLM-5.2 golden thinking-on +# acceptance length for three speculative steps. + +model: + path: "hf:zai-org/GLM-5.2-FP8" + container: "lmsysorg/sglang:v0.5.16-cu130" + precision: "fp8" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +resources: + gpu_type: h200 + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 2 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + prefill_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + decode_environment: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + SGLANG_SIMULATE_ACC_LEN: "2.99" + SGLANG_SIMULATE_ACC_METHOD: "match-expected" + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + + sglang_config: + prefill: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 8 + ep-size: 1 + attn-cp-size: 8 + enable-prefill-cp: true + cp-strategy: interleave + enable-dsa-cache-layer-split: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + kv-cache-dtype: fp8_e4m3 + context-length: 1048576 + max-total-tokens: 1048576 + chunked-prefill-size: 32768 + mem-fraction-static: 0.85 + max-running-requests: 32 + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + enable-cache-report: true + + decode: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 8 + dp-size: 8 + ep-size: 1 + enable-dp-attention: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + kv-cache-dtype: fp8_e4m3 + dsa-decode-backend: flashmla_kv + context-length: 1048576 + max-total-tokens: 1048576 + mem-fraction-static: 0.85 + max-running-requests: 200 + page-size: 64 + disable-radix-cache: true + speculative-algorithm: EAGLE + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + enable-cache-report: true + +sbatch_directives: + mem: "0" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e5ca6d0f94..a222b0c972 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7609,6 +7609,73 @@ qwen3.5-fp8-gb300-dynamo-sglang: ep: 16 dp-attn: true +# GLM-5.2 FP8 full-context AgentX refresh on H200. This preserves the 2P2D +# PCP8/TP8-DP8 topology and c8/c12/c16 curve from Actions run 30133535261 while +# enabling native EAGLE MTP with golden AL and GPU-resident decode KV cache. +glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: + image: lmsysorg/sglang:v0.5.16-cu130 + model: zai-org/GLM-5.2-FP8 + model-prefix: glm5.2 + runner: cluster:h200-dgxc + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.3.0.dev1" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: mtp + conc-list: [8] + kv-offloading: none + prefill: + num-worker: 2 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: true + - spec-decoding: mtp + conc-list: [12] + kv-offloading: none + prefill: + num-worker: 2 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: true + - spec-decoding: mtp + conc-list: [16] + kv-offloading: none + prefill: + num-worker: 2 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: true + # GLM-5.2 B300 NVFP4 AgentX with EAGLE/MTP speculative decoding, following the # AgentX speculative-decoding policy in MODELS.md. SGLang EAGLE runs off # GLM-5.2's built-in nextn head (num-steps 3, diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f4cae47e59..46490e50e7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5665,3 +5665,15 @@ - "Refresh submission with up to date AgentX harness." - "Capture the vLLM backend Prometheus endpoint for every topology and require vllm: metrics before publishing AgentX results." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2516 + +- config-keys: + - glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d + scenario-type: + - agentic-coding + description: + - "Refresh submission with up to date AgentX harness." + - "Retain the 2P2D PCP8 prefill, TP8/DP8 decode, Mooncake, 1M-context topology, and c8/c12/c16 search space; use GPU-resident decode KV cache because released SGLang does not support combining HiSparse with MTP" + - "Use EAGLE with three speculative steps, top-k 1, four draft tokens, and the committed GLM-5.2 thinking-on golden synthetic acceptance length of 2.99; evals retain real verification" + - "Use lmsysorg/sglang:v0.5.16-cu130 and the latest NVIDIA/srt-slurm release, v1.0.44, which includes logical-worker Prometheus URL injection for AIPerf" + - "Enable SGLang cache reporting on both prefill and decode workers so AIPerf records prompt-cache read-token usage alongside all logical-worker Prometheus metrics" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2529 diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 29c77b13cd..8991e04182 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -11,6 +11,13 @@ source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" if [[ "$IS_MULTINODE" == "true" ]]; then + if [[ -z "${CONFIG_FILE:-}" ]]; then + echo "Error: CONFIG_FILE is not set. The srt-slurm path requires a CONFIG_FILE in additional-settings." >&2 + exit 1 + fi + CONFIG_PATH="${CONFIG_FILE%%:*}" + LOCAL_CONFIG_FILE="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" + # MODEL_PATH: Override with pre-downloaded paths on H200 runner # The yaml files specify HuggingFace model IDs for portability, but we use # local paths to avoid repeated downloading on the shared H200 cluster. @@ -18,6 +25,12 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/models/DeepSeek-R1-0528" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" + elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp8" ]]; then + export MODEL_PATH="${GLM52_FP8_MODEL_PATH:-/models/GLM-5.2-FP8}" + if [[ ! -d "$MODEL_PATH" ]]; then + export MODEL_PATH="hf:zai-org/GLM-5.2-FP8" + fi + export SRT_SLURM_MODEL_PREFIX="glm5.2-fp8" else echo "Unsupported model prefix/precision for dynamo-sglang: $MODEL_PREFIX/$PRECISION" exit 1 @@ -51,7 +64,12 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -rf "$SRT_REPO_DIR" fi - if [[ $IS_AGENTIC == "1" && $FRAMEWORK == "vllm" && $MODEL_PREFIX == "kimik3" ]]; then + if [[ $IS_AGENTIC == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" ]]; then + # v1.0.44 includes the AgentX custom benchmark integration and passes + # every logical SGLang worker's Prometheus URL to AIPerf. + git clone --branch v1.0.44 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + elif [[ $IS_AGENTIC == "1" && $FRAMEWORK == "vllm" && $MODEL_PREFIX == "kimik3" ]]; then git clone https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 @@ -87,7 +105,11 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ $FRAMEWORK == "dynamo-sglang" ]]; then # SGLang container mapping - SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/+/g').sqsh" + if [[ $MODEL_PREFIX == "glm5.2" ]]; then + SQUASH_FILE="/data/gharunners/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + else + SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/+/g').sqsh" + fi CONTAINER_KEY="$IMAGE" elif [[ $FRAMEWORK == "dynamo-trt" ]]; then # TRT-LLM container mapping - convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) @@ -98,6 +120,26 @@ if [[ "$IS_MULTINODE" == "true" ]]; then SQUASH_FILE="/data/gharunners/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" fi + if [[ $MODEL_PREFIX == "glm5.2" ]] && ! unsquashfs -l "$SQUASH_FILE" >/dev/null 2>&1; then + DOCKER_IMAGE=$(echo "$IMAGE" | sed 's/#/\//g') + LOCK_FILE="${SQUASH_FILE}.lock" + mkdir -p "$(dirname "$SQUASH_FILE")" + srun --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \ + --nodes=1 --ntasks=1 --time=30 --job-name="$RUNNER_NAME" \ + bash -c " + set -euo pipefail + exec 9>\"$LOCK_FILE\" + flock -w 1800 9 + if unsquashfs -l \"$SQUASH_FILE\" >/dev/null 2>&1; then + exit 0 + fi + rm -f \"$SQUASH_FILE\" + export ENROOT_CACHE_PATH=\${HOME}/.cache/enroot + mkdir -p \"\$ENROOT_CACHE_PATH\" + enroot import -o \"$SQUASH_FILE\" docker://$DOCKER_IMAGE + " + fi + export ISL="$ISL" export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" @@ -150,22 +192,25 @@ EOF echo "Running make setup..." make setup ARCH=x86_64 + if [[ -f "$LOCAL_CONFIG_FILE" ]]; then + mkdir -p "$(dirname "$CONFIG_PATH")" + cp "$LOCAL_CONFIG_FILE" "$CONFIG_PATH" + fi + # Export eval-related env vars for srt-slurm post-benchmark eval export INFMAX_WORKSPACE="$GITHUB_WORKSPACE" echo "Submitting job with srtctl..." - if [[ -z "$CONFIG_FILE" ]]; then - echo "Error: CONFIG_FILE is not set. The srt-slurm path requires a CONFIG_FILE in additional-settings." >&2 - echo "Config: MODEL_PREFIX=${MODEL_PREFIX} PRECISION=${PRECISION} FRAMEWORK=${FRAMEWORK}" >&2 - exit 1 - fi - # Override the job name in the config file with the runner name - sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE" - sed -i '/^health_check:/,/^[^ ]/{ /^health_check:/d; /^ /d; }' "${CONFIG_FILE%%:*}" - printf '\nhealth_check:\n max_attempts: 720\n interval_seconds: 10\n' >> "${CONFIG_FILE%%:*}" - SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) + sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" + sed -i '/^health_check:/,/^[^ ]/{ /^health_check:/d; /^ /d; }' "$CONFIG_PATH" + printf '\nhealth_check:\n max_attempts: 720\n interval_seconds: 10\n' >> "$CONFIG_PATH" + WORKLOAD_TAG="${ISL}x${OSL}" + if [[ "$IS_AGENTIC" == "1" ]]; then + WORKLOAD_TAG="agentic" + fi + SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h200,${MODEL_PREFIX},${PRECISION},${WORKLOAD_TAG},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output