From 0555089fc48ed3a943b315a828cbf6bf9e6fa5d2 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 10 Aug 2026 15:38:38 -0700 Subject: [PATCH 1/2] feat(config): add DSV4 B200 nscale UCX canary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 DSV4 B200 nscale UCX 验证配置。 --- ...agg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml | 147 ++++++++++++++++++ configs/nvidia-master.yaml | 31 ++++ perf-changelog.yaml | 8 + runners/launch_b200-nscale-slurm.sh | 26 +++- 4 files changed, 205 insertions(+), 7 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..ea93b16f83 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml @@ -0,0 +1,147 @@ +name: "disagg-b200-8k1k-mid-curve-1p2d-dep8-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 8 + decode_nodes: 2 + decode_workers: 2 + gpus_per_decode: 8 + +health_check: + max_attempts: 240 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_CUMEM_ENABLE: "1" + # Keep RC for inter-node transport, but stage CUDA buffers through host memory. + UCX_TLS: "rc,cuda_ipc,cuda_copy,self" + UCX_IB_GPU_DIRECT_RDMA: "no" + UCX_PROTO_EMULATION_ENABLE: "y" + UCX_PROTO_INFO: "y" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_CUMEM_ENABLE: "1" + # Keep RC for inter-node transport, but stage CUDA buffers through host memory. + UCX_TLS: "rc,cuda_ipc,cuda_copy,self" + UCX_IB_GPU_DIRECT_RDMA: "no" + UCX_PROTO_EMULATION_ENABLE: "y" + UCX_PROTO_INFO: "y" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.8 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.85 + max-running-requests: 1024 + cuda-graph-max-bs: 512 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 263dec901e..f97b749a9d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5280,6 +5280,37 @@ dsv4-fp4-gb300-dynamo-vllm: ep: 16 dp-attn: true +dsv4-fp4-b200-dynamo-sglang-mtp: + image: lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: b200-nscale + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [256] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + dsv4-fp4-b200-dynamo-vllm-mtp: image: vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 64f87b4479..94b6abea65 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5770,3 +5770,11 @@ - "models.yaml: add mtp_flags (--speculative-algorithm EAGLE --speculative-eagle-topk 1) to DeepSeek-V4-Pro-AgentX; num-steps/num-draft-tokens are derived from DECODE_MTP_SIZE in server_sglang.sh. DeepSeek-V4's MTP head is a native NextN/EAGLE draft shipped with the model, so no --speculative-draft-model-path is needed." - "Two search-space arms: TP8/EP1 no-DP at conc-list [2,4,8,16,32], and TP8/EP8/DPA (ep=8, dp-attn=true) at conc-list [64,96,128], both on image lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2309 + +- config-keys: + - dsv4-fp4-b200-dynamo-sglang-mtp + description: + - "Add a focused DeepSeek-V4-Pro FP4 Dynamo-SGLang MTP configuration on the B200 nscale runner." + - "Keep UCX RC transport while disabling GPU-direct RDMA registration and enabling host-staged protocol emulation for CUDA buffers." + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2554 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index b5a448d307..349cc5c9d7 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -8,8 +8,9 @@ # them would force every dgxc path to become an override hook and would let a # dgxc-only change silently alter nscale runs. # -# Scope: multi-node Dynamo-vLLM DeepSeek-V4-Pro and Kimi K2.6 FP4 runs on -# the b200-nscale/b200-new runner labels. Anything else exits non-zero. +# Scope: multi-node Dynamo-vLLM DeepSeek-V4-Pro and Kimi K2.6 FP4 runs, plus +# DeepSeek-V4-Pro FP4 Dynamo-SGLang MTP, on the b200-nscale/b200-new runner +# labels. Anything else exits non-zero. SLURM_PARTITION="batch_1" SLURM_ACCOUNT="benchmark" @@ -49,8 +50,9 @@ else exit 1 fi -if [[ $FRAMEWORK != "dynamo-vllm" ]]; then - echo "Unsupported framework for b200-nscale: $FRAMEWORK (only dynamo-vllm)" >&2 +if [[ $FRAMEWORK != "dynamo-vllm" ]] && + [[ $MODEL_PREFIX != "dsv4" || $PRECISION != "fp4" || $FRAMEWORK != "dynamo-sglang" || $SPEC_DECODING != "mtp" ]]; then + echo "Unsupported framework/configuration for b200-nscale: $MODEL_PREFIX/$PRECISION/$FRAMEWORK/$SPEC_DECODING" >&2 exit 1 fi @@ -59,7 +61,15 @@ export SERVED_MODEL_NAME=$MODEL echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" rm -rf "$SRT_REPO_DIR" -if [[ $MODEL_PREFIX == "dsv4" ]]; then +if [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then + git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + # Pin srt-slurm so this focused run changes only the runner and UCX + # transport behavior. + git checkout 04e87fcc505d6d851451781a5499ca19a02ec2b4 || exit 1 + mkdir -p recipes/sglang/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 +elif [[ $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 git checkout aflowers/vllm-gb200-v0.20.0 || exit 1 @@ -165,6 +175,7 @@ model_paths: # Container aliases containers: dynamo-vllm: "${SQUASH_FILE}" + dynamo-sglang: "${SQUASH_FILE}" "${IMAGE}": "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" use_exclusive_sbatch_directive: true @@ -201,8 +212,9 @@ sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "$CONFIG_PATH" inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 SRTCTL_PREFLIGHT_ARGS=() -# Kimi K2.6 weights are staged on the Slurm compute nodes, not the login node. -if [[ $MODEL_PREFIX == "kimik2.6" ]]; then +# These weights are staged on the Slurm compute nodes, not the login node. +if [[ $MODEL_PREFIX == "kimik2.6" ]] || + [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then SRTCTL_PREFLIGHT_ARGS+=(--no-preflight) fi From 8c15a0af83780bfab26d716311d1b37665b14c54 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 10 Aug 2026 19:12:05 -0700 Subject: [PATCH 2/2] feat(config): expand DSV4 B200 Dynamo-SGLang MTP matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 扩展 DSV4 B200 Dynamo-SGLang MTP 配置，补充五个分离式拓扑并保持配置与 recipe 一致。 --- ...agg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml | 120 ++++++++++++++++ ...gg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml | 127 +++++++++++++++++ ...agg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml | 128 ++++++++++++++++++ ...agg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml | 12 +- ...agg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml | 128 ++++++++++++++++++ configs/nvidia-master.yaml | 56 ++++++++ perf-changelog.yaml | 4 +- runners/launch_b200-nscale-slurm.sh | 3 +- 8 files changed, 564 insertions(+), 14 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml new file mode 100644 index 0000000000..685537c6a6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml @@ -0,0 +1,120 @@ +name: "disagg-b200-8k1k-low-latency-1p1d-tp8-tp8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 8 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +health_check: + max_attempts: 240 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 16 + cuda-graph-max-bs: 8 + chunked-prefill-size: 65536 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 8 + cuda-graph-max-bs: 8 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "1" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml new file mode 100644 index 0000000000..18b377b863 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml @@ -0,0 +1,127 @@ +name: "disagg-b200-8k1k-low-latency-1p6d-dep8-tp8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 8 + decode_nodes: 6 + decode_workers: 6 + gpus_per_decode: 8 + +health_check: + max_attempts: 240 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + mem-fraction-static: 0.9 + max-running-requests: 256 + cuda-graph-max-bs: 128 + chunked-prefill-size: 65536 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + context-length: 16384 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "32x64x128" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..ea30d07e55 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml @@ -0,0 +1,128 @@ +name: "disagg-b200-8k1k-mid-curve-1p1d-dep8-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 8 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 + +health_check: + max_attempts: 240 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + mem-fraction-static: 0.8 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + mem-fraction-static: 0.9 + max-running-requests: 1536 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "256x1024" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml index ea93b16f83..9a6254e989 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml @@ -48,11 +48,7 @@ backend: SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" NCCL_CUMEM_ENABLE: "1" - # Keep RC for inter-node transport, but stage CUDA buffers through host memory. - UCX_TLS: "rc,cuda_ipc,cuda_copy,self" - UCX_IB_GPU_DIRECT_RDMA: "no" - UCX_PROTO_EMULATION_ENABLE: "y" - UCX_PROTO_INFO: "y" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" @@ -71,11 +67,7 @@ backend: SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" NCCL_CUMEM_ENABLE: "1" - # Keep RC for inter-node transport, but stage CUDA buffers through host memory. - UCX_TLS: "rc,cuda_ipc,cuda_copy,self" - UCX_IB_GPU_DIRECT_RDMA: "no" - UCX_PROTO_EMULATION_ENABLE: "y" - UCX_PROTO_INFO: "y" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml new file mode 100644 index 0000000000..e5d9a708a1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml @@ -0,0 +1,128 @@ +name: "disagg-b200-8k1k-mid-curve-5p3d-dep8-dep8-mtp" + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + +dynamo: + hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" + install: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + precision: "fp4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 5 + prefill_workers: 5 + gpus_per_prefill: 8 + decode_nodes: 3 + decode_workers: 3 + gpus_per_decode: 8 + +health_check: + max_attempts: 240 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_RADIX_FORCE_MISS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" + SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" + NCCL_CUMEM_ENABLE: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "prefill" + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + mem-fraction-static: 0.8 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + chunked-prefill-size: 65536 + stream-interval: 60 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + tool-call-parser: deepseekv4 + disaggregation-mode: "decode" + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-a2a-backend: "megamoe" + deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + speculative-algo: "EAGLE" + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + mem-fraction-static: 0.9 + max-running-requests: 2560 + cuda-graph-max-bs: 1024 + swa-full-tokens-ratio: 0.15 + context-length: 16384 + stream-interval: 60 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "6144" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f97b749a9d..de3b9505a5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5296,6 +5296,48 @@ dsv4-fp4-b200-dynamo-sglang-mtp: - isl: 8192 osl: 1024 search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [32, 64, 128] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml" + decode: + num-worker: 6 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [256, 1024] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true - spec-decoding: "mtp" conc-list: [256] prefill: @@ -5310,6 +5352,20 @@ dsv4-fp4-b200-dynamo-sglang-mtp: tp: 8 ep: 8 dp-attn: true + - spec-decoding: "mtp" + conc-list: [6144] + prefill: + num-worker: 5 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true dsv4-fp4-b200-dynamo-vllm-mtp: image: vllm/vllm-openai:vllm-x86_64-cu13-0.25.1-7a33ba9 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 94b6abea65..e04d2dd430 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5774,7 +5774,7 @@ - config-keys: - dsv4-fp4-b200-dynamo-sglang-mtp description: - - "Add a focused DeepSeek-V4-Pro FP4 Dynamo-SGLang MTP configuration on the B200 nscale runner." - - "Keep UCX RC transport while disabling GPU-direct RDMA registration and enabling host-staged protocol emulation for CUDA buffers." + - "Add a DeepSeek-V4-Pro FP4 Dynamo-SGLang MTP configuration for B200 with five 8k/1k disaggregated topologies." + - "Use NIXL for KV transfer with EAGLE speculative decoding and chat-formatted benchmark inputs." - "Image: lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2554 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index 349cc5c9d7..9b9a1e3f96 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -64,8 +64,7 @@ rm -rf "$SRT_REPO_DIR" if [[ $MODEL_PREFIX == "dsv4" && $FRAMEWORK == "dynamo-sglang" ]]; then git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 - # Pin srt-slurm so this focused run changes only the runner and UCX - # transport behavior. + # Pin the srt-slurm revision used by these checked-in recipes. git checkout 04e87fcc505d6d851451781a5499ca19a02ec2b4 || exit 1 mkdir -p recipes/sglang/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4