diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index 192e0f5b95..9559ffb75b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -1,8 +1,8 @@ name: "svf-vllm-agg-gb300-tp4-mtp-agentic" # GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode at concurrency 4. Scheduler, CUDA-graph, -# and memory settings match the B300 vLLM TP4 MTP agentic configuration. +# and serves both prefill and decode at concurrency 8. Size max-num-seqs at +# 4x concurrency and expand the MTP CUDA-graph envelope to match. model: path: "deepseek-v4-pro" @@ -77,7 +77,7 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "0" TORCH_SYMMMEM: "NVSHMEM" @@ -110,17 +110,16 @@ backend: tensor-parallel-size: 4 pipeline-parallel-size: 1 disable-custom-all-reduce: true - enable-cumem-allocator: true attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 - max-num-seqs: 16 + max-num-seqs: 32 max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' - gpu-memory-utilization: 0.93 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" @@ -141,6 +140,8 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index 6a950787fe..4eebce98fa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -1,7 +1,8 @@ name: "svf-vllm-agg-gb300-tp8-mtp-agentic" -# Validated GB300 AgentX aggregate topology: one TP8 worker spans two -# four-GPU nodes and serves both prefill and decode at concurrency 1. +# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 4. Keep at least +# 16 sequence slots and otherwise size the scheduler at 4x concurrency. model: path: "deepseek-v4-pro" @@ -77,7 +78,7 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "0" TORCH_SYMMMEM: "NVSHMEM" @@ -110,18 +111,16 @@ backend: tensor-parallel-size: 8 pipeline-parallel-size: 1 disable-custom-all-reduce: true - enable-cumem-allocator: true attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 - max-num-seqs: 32 + max-num-seqs: 16 max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 128 - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' - gpu-memory-utilization: 0.90 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml similarity index 85% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml index dd58dde28f..e5cc1c76ea 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml @@ -1,7 +1,7 @@ -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic" +name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic" -# Validated GB300 AgentX MTP3 low-latency topology: one DEP4 prefill worker -# feeds one DEP8 decode worker at concurrency 128. +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one +# DEP8 decode worker at concurrency 256. model: path: "deepseek-v4-pro" @@ -17,7 +17,7 @@ identity: dynamo: "1.3.0.dev20260720" dynamo: - version: "1.3.0.dev20260720" + wheel: "1.3.0.dev20260720" install: true setup_script: vllm-container-deps.sh @@ -57,6 +57,8 @@ frontend: router-session-affinity-ttl-secs: 900 env: DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" backend: type: vllm @@ -65,7 +67,7 @@ backend: mooncake_kv_store: store_config: metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" + global_segment_size: "180GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" @@ -86,6 +88,7 @@ backend: max-model-len: 1048576 max-num-seqs: 64 max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 @@ -94,12 +97,12 @@ backend: no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' moe-backend: "deep_gemm_amxf4_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" kv-cache-dtype: "fp8" tensor-parallel-size: 1 @@ -121,7 +124,7 @@ backend: no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' moe-backend: "deep_gemm_amxf4_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] @@ -149,7 +152,7 @@ backend: VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" @@ -176,7 +179,7 @@ backend: VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" @@ -198,6 +201,8 @@ benchmark: IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml similarity index 79% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml index 00a876ebe8..d5a0b44161 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml @@ -1,7 +1,8 @@ -name: "svf-vllm-disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic" +name: "svf-vllm-disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic" -# Validated GB300 AgentX MTP3 efficiency topology: one DEP8 prefill worker and -# one DEP8 decode worker at concurrency 384. +# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP12 decode worker at concurrency 1152. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. model: path: "deepseek-v4-pro" @@ -17,9 +18,13 @@ identity: dynamo: "1.3.0.dev20260720" dynamo: - version: "1.3.0.dev20260720" + wheel: "1.3.0.dev20260720" install: true +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + setup_script: vllm-container-deps.sh slurm: @@ -34,21 +39,17 @@ resources: gpus_per_node: 4 het_jobs: false spread_workers: false - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 + prefill_nodes: 4 + decode_nodes: 3 + prefill_workers: 2 decode_workers: 1 gpus_per_prefill: 8 - gpus_per_decode: 8 + gpus_per_decode: 12 infra: etcd_nats_dedicated_node: false nats_max_payload_mb: 32 -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - frontend: type: dynamo enable_multiple_frontends: false @@ -57,6 +58,8 @@ frontend: router-session-affinity-ttl-secs: 900 env: DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" backend: type: vllm @@ -65,7 +68,7 @@ backend: mooncake_kv_store: store_config: metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" + global_segment_size: "180GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" @@ -84,44 +87,44 @@ backend: enable-expert-parallel: true enable-ep-weight-filter: true max-model-len: 1048576 - max-num-seqs: 32 + max-num-seqs: 64 max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 - max-cudagraph-capture-size: 128 - gpu-memory-utilization: 0.92 + gpu-memory-utilization: 0.90 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' moe-backend: "deep_gemm_amxf4_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 - data-parallel-size: 8 + data-parallel-size: 12 data-parallel-rpc-port: 13345 enable-cumem-allocator: true enable-expert-parallel: true enable-ep-weight-filter: true max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 1024 + max-num-seqs: 64 + max-num-batched-tokens: 256 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - gpu-memory-utilization: 0.92 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' moe-backend: "deep_gemm_amxf4_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] @@ -136,6 +139,7 @@ backend: VLLM_SERVER_DEV_MODE: "1" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" UCX_MEMTYPE_CACHE: "n" UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" @@ -143,13 +147,13 @@ backend: NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" @@ -172,11 +176,11 @@ backend: NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep12-decode-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" @@ -199,6 +203,8 @@ benchmark: IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml new file mode 100644 index 0000000000..f770241932 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml @@ -0,0 +1,210 @@ +name: "svf-vllm-disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic" + +# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP16 decode worker at concurrency 1024. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + wheel: "1.3.0.dev20260720" + install: true + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 4 + decode_nodes: 4 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep16-decode-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c7a82fdb1a..0948ed6f9d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7691,6 +7691,8 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: scenarios: agentic-coding: - search-space: + # Keep the checked-in recipes on real MTP verification. Throughput jobs + # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - spec-decoding: mtp conc-list: [1] prefill: @@ -7699,6 +7701,8 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" # The aggregate worker also performs decode; keep the decode worker # count at zero so result aggregation counts eight GPUs only once. @@ -7709,12 +7713,32 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: dp-attn: false - spec-decoding: mtp conc-list: [4] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts eight GPUs only once. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [8] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" # The aggregate worker also performs decode; keep the decode worker # count at zero so result aggregation counts four GPUs only once. @@ -7737,31 +7761,54 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: scenarios: agentic-coding: - search-space: + # Keep the checked-in recipes on real MTP verification. Throughput jobs + # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - spec-decoding: mtp - conc-list: [128] + conc-list: [1152] router: { name: dynamo-router, version: "1.3.0.dev20260720" } prefill: - num-worker: 1 - tp: 4 - ep: 4 + num-worker: 2 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml" decode: num-worker: 1 + tp: 12 + ep: 12 + dp-attn: true + - spec-decoding: mtp + conc-list: [1024] + router: { name: dynamo-router, version: "1.3.0.dev20260720" } + prefill: + num-worker: 2 tp: 8 ep: 8 dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true - spec-decoding: mtp - conc-list: [384] + conc-list: [256] router: { name: dynamo-router, version: "1.3.0.dev20260720" } prefill: num-worker: 1 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml" decode: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 228b327a4a..067b8a79ff 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5971,3 +5971,13 @@ - "Compare the B200 TP8 and DEP8 concurrency range and extend throughput with 1P1D and 2P1D DEP8 disaggregation." - "Use KV-aware Dynamo routing with 4-hour correlation-ID affinity, authoritative vLLM KV events, KV-cache token metrics, and every logical vLLM server-metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2567 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Inject the committed synthetic MTP acceptance length only for GB300 AgentX throughput; keep eval-only jobs on real target verification." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2571 + diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 27154c404e..54d208bc7f 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -4,6 +4,9 @@ set -exo pipefail +# shellcheck source=runners/slurm_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" + export SLURM_PARTITION="batch_1" export SLURM_ACCOUNT="benchmark" export ENROOT_ROOTFS_WRITABLE=1 @@ -425,6 +428,11 @@ fi CONFIG_PATH="${CONFIG_FILE%%:*}" sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" +# Throughput recipes opt into synthetic acceptance through the master config. +# Eval-only jobs leave the checked-in real-MTP recipe unchanged so generated +# tokens still pass target-model verification. +inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 + # --no-preflight skips srtctl's pre-submit model-path stat, which runs on # the GHA runner host (im-gb300-login-02, an x86 login node). It's required # whenever model.path resolves to the node-local /scratch NVMe that the login