[AgentX] Tune DSV4 GB300 MTP / 调优 DSV4 GB300 MTP

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml

@@ -1,8 +1,8 @@
 name: "svf-vllm-agg-gb300-tp4-mtp-agentic"
 
 # GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node
-# and serves both prefill and decode at concurrency 4. Scheduler, CUDA-graph,
-# and memory settings match the B300 vLLM TP4 MTP agentic configuration.
+# and serves both prefill and decode at concurrency 8. Size max-num-seqs at
+# 4x concurrency and expand the MTP CUDA-graph envelope to match.
 
 model:
   path: "deepseek-v4-pro"
@@ -77,7 +77,7 @@ backend:
     VLLM_ENGINE_READY_TIMEOUT_S: "3600"
     VLLM_RPC_TIMEOUT: "600000"
     VLLM_LOG_STATS_INTERVAL: "1"
-    VLLM_V2_WARMUP_MAX_NUM_SEQS: "16"
+    VLLM_V2_WARMUP_MAX_NUM_SEQS: "32"
     TILELANG_CLEANUP_TEMP_FILES: "1"
     VLLM_USE_NCCL_SYMM_MEM: "0"
     TORCH_SYMMMEM: "NVSHMEM"
@@ -110,17 +110,16 @@ backend:
       tensor-parallel-size: 4
       pipeline-parallel-size: 1
       disable-custom-all-reduce: true
-      enable-cumem-allocator: true
       attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
       max-model-len: 1048576
-      max-num-seqs: 16
+      max-num-seqs: 32
       max-num-batched-tokens: 8192
       trust-remote-code: true
       no-enable-flashinfer-autotune: true
       block-size: 256
-      compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}'
-      speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
-      gpu-memory-utilization: 0.93
+      compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}'
+      speculative-config: '{"method":"mtp","num_speculative_tokens":3}'
+      gpu-memory-utilization: 0.94
       stream-interval: 10
       no-disable-hybrid-kv-cache-manager: true
       tokenizer-mode: "deepseek_v4"
@@ -141,6 +140,8 @@ benchmark:
     PORT: "8000"
     IS_MULTINODE: "true"
     AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    # Avoid concurrent readers observing a mismatched mmap data/index pair.
+    AIPERF_DATASET_MMAP_CACHE_ENABLED: "false"
     AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
     HF_HUB_CACHE: "/hf_hub_cache"
     WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml

@@ -1,7 +1,8 @@
 name: "svf-vllm-agg-gb300-tp8-mtp-agentic"
 
-# Validated GB300 AgentX aggregate topology: one TP8 worker spans two
-# four-GPU nodes and serves both prefill and decode at concurrency 1.
+# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU
+# nodes and serves both prefill and decode at concurrency 4. Keep at least
+# 16 sequence slots and otherwise size the scheduler at 4x concurrency.
 
 model:
   path: "deepseek-v4-pro"
@@ -77,7 +78,7 @@ backend:
     VLLM_ENGINE_READY_TIMEOUT_S: "3600"
     VLLM_RPC_TIMEOUT: "600000"
     VLLM_LOG_STATS_INTERVAL: "1"
-    VLLM_V2_WARMUP_MAX_NUM_SEQS: "32"
+    VLLM_V2_WARMUP_MAX_NUM_SEQS: "16"
     TILELANG_CLEANUP_TEMP_FILES: "1"
     VLLM_USE_NCCL_SYMM_MEM: "0"
     TORCH_SYMMMEM: "NVSHMEM"
@@ -110,18 +111,16 @@ backend:
       tensor-parallel-size: 8
       pipeline-parallel-size: 1
       disable-custom-all-reduce: true
-      enable-cumem-allocator: true
       attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
       max-model-len: 1048576
-      max-num-seqs: 32
+      max-num-seqs: 16
       max-num-batched-tokens: 8192
       trust-remote-code: true
       no-enable-flashinfer-autotune: true
       block-size: 256
-      compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
-      max-cudagraph-capture-size: 128
-      speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
-      gpu-memory-utilization: 0.90
+      compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}'
+      speculative-config: '{"method":"mtp","num_speculative_tokens":3}'
+      gpu-memory-utilization: 0.94
       stream-interval: 10
       no-disable-hybrid-kv-cache-manager: true
       tokenizer-mode: "deepseek_v4"

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml → benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml

@@ -1,7 +1,7 @@
-name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic"
+name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic"
 
-# Validated GB300 AgentX MTP3 low-latency topology: one DEP4 prefill worker
-# feeds one DEP8 decode worker at concurrency 128.
+# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one
+# DEP8 decode worker at concurrency 256.
 
 model:
   path: "deepseek-v4-pro"
@@ -17,7 +17,7 @@ identity:
     dynamo: "1.3.0.dev20260720"
 
 dynamo:
-  version: "1.3.0.dev20260720"
+  wheel: "1.3.0.dev20260720"
   install: true
 
 setup_script: vllm-container-deps.sh
@@ -57,6 +57,8 @@ frontend:
     router-session-affinity-ttl-secs: 900
   env:
     DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600"
+    DYN_TCP_CHANNEL_BUFFER: "128"
+    DYN_TCP_REQUEST_TIMEOUT: "60"
 
 backend:
   type: vllm
@@ -65,7 +67,7 @@ backend:
   mooncake_kv_store:
     store_config:
       metadata_server: "P2PHANDSHAKE"
-      global_segment_size: "150GB"
+      global_segment_size: "180GB"
       local_buffer_size: "4GB"
       protocol: "rdma"
       device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
@@ -86,6 +88,7 @@ backend:
       max-model-len: 1048576
       max-num-seqs: 64
       max-num-batched-tokens: 8192
+      long-prefill-token-threshold: 1024
       trust-remote-code: true
       no-enable-flashinfer-autotune: true
       block-size: 256
@@ -94,12 +97,12 @@ backend:
       no-disable-hybrid-kv-cache-manager: true
       tokenizer-mode: "deepseek_v4"
       attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
-      speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
+      speculative-config: '{"method":"mtp","num_speculative_tokens":3}'
       moe-backend: "deep_gemm_amxf4_mega_moe"
       numa-bind: true
       numa-bind-nodes: [0, 0, 1, 1]
     decode:
-      kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}'
+      kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}'
       served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
       kv-cache-dtype: "fp8"
       tensor-parallel-size: 1
@@ -121,7 +124,7 @@ backend:
       no-disable-hybrid-kv-cache-manager: true
       tokenizer-mode: "deepseek_v4"
       attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
-      speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
+      speculative-config: '{"method":"mtp","num_speculative_tokens":3}'
       moe-backend: "deep_gemm_amxf4_mega_moe"
       numa-bind: true
       numa-bind-nodes: [0, 0, 1, 1]
@@ -149,7 +152,7 @@ backend:
     VLLM_USE_BREAKABLE_CUDAGRAPH: "0"
     VLLM_CONNECTOR_PREFETCH_DEPTH: "8"
     VLLM_DSV4_MEGA_FP8_COMBINE: "1"
-    DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}"
+    DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}"
     MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
     MC_STORE_CLIENT_METRIC: "1"
     MC_STORE_CLIENT_METRIC_INTERVAL: "5"
@@ -176,7 +179,7 @@ backend:
     VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
     VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
     VLLM_DSV4_MEGA_FP8_COMBINE: "1"
-    DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}"
+    DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}"
     MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
     MC_STORE_CLIENT_METRIC: "1"
     MC_STORE_CLIENT_METRIC_INTERVAL: "5"
@@ -198,6 +201,8 @@ benchmark:
     IS_MULTINODE: "true"
     AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
     AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
+    # Avoid concurrent readers observing a mismatched mmap data/index pair.
+    AIPERF_DATASET_MMAP_CACHE_ENABLED: "false"
     AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
     HF_HUB_CACHE: "/hf_hub_cache"
     WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml → benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml

@@ -1,7 +1,8 @@
-name: "svf-vllm-disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic"
+name: "svf-vllm-disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic"
 
-# Validated GB300 AgentX MTP3 efficiency topology: one DEP8 prefill worker and
-# one DEP8 decode worker at concurrency 384.
+# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one
+# DEP12 decode worker at concurrency 1152. Decode consumes P/D KV through NIXL
+# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead.
 
 model:
   path: "deepseek-v4-pro"
@@ -17,9 +18,13 @@ identity:
     dynamo: "1.3.0.dev20260720"
 
 dynamo:
-  version: "1.3.0.dev20260720"
+  wheel: "1.3.0.dev20260720"
   install: true
 
+environment:
+  # Mooncake prefix-block hashes must match across processes and nodes.
+  PYTHONHASHSEED: "0"
+
 setup_script: vllm-container-deps.sh
 
 slurm:
@@ -34,21 +39,17 @@ resources:
   gpus_per_node: 4
   het_jobs: false
   spread_workers: false
-  prefill_nodes: 2
-  decode_nodes: 2
-  prefill_workers: 1
+  prefill_nodes: 4
+  decode_nodes: 3
+  prefill_workers: 2
   decode_workers: 1
   gpus_per_prefill: 8
-  gpus_per_decode: 8
+  gpus_per_decode: 12
 
 infra:
   etcd_nats_dedicated_node: false
   nats_max_payload_mb: 32
 
-environment:
-  # Mooncake prefix-block hashes must match across processes and nodes.
-  PYTHONHASHSEED: "0"
-
 frontend:
   type: dynamo
   enable_multiple_frontends: false
@@ -57,6 +58,8 @@ frontend:
     router-session-affinity-ttl-secs: 900
   env:
     DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600"
+    DYN_TCP_CHANNEL_BUFFER: "128"
+    DYN_TCP_REQUEST_TIMEOUT: "60"
 
 backend:
   type: vllm
@@ -65,7 +68,7 @@ backend:
   mooncake_kv_store:
     store_config:
       metadata_server: "P2PHANDSHAKE"
-      global_segment_size: "150GB"
+      global_segment_size: "180GB"
       local_buffer_size: "4GB"
       protocol: "rdma"
       device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
@@ -84,44 +87,44 @@ backend:
       enable-expert-parallel: true
       enable-ep-weight-filter: true
       max-model-len: 1048576
-      max-num-seqs: 32
+      max-num-seqs: 64
       max-num-batched-tokens: 8192
+      long-prefill-token-threshold: 1024
       trust-remote-code: true
       no-enable-flashinfer-autotune: true
       block-size: 256
-      max-cudagraph-capture-size: 128
-      gpu-memory-utilization: 0.92
+      gpu-memory-utilization: 0.90
       no-disable-hybrid-kv-cache-manager: true
       tokenizer-mode: "deepseek_v4"
       attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
-      speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
+      speculative-config: '{"method":"mtp","num_speculative_tokens":3}'
       moe-backend: "deep_gemm_amxf4_mega_moe"
       numa-bind: true
       numa-bind-nodes: [0, 0, 1, 1]
     decode:
-      kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}'
+      kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}'
       served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
       kv-cache-dtype: "fp8"
       tensor-parallel-size: 1
       pipeline-parallel-size: 1
-      data-parallel-size: 8
+      data-parallel-size: 12
       data-parallel-rpc-port: 13345
       enable-cumem-allocator: true
       enable-expert-parallel: true
       enable-ep-weight-filter: true
       max-model-len: 1048576
-      max-num-seqs: 256
-      max-num-batched-tokens: 1024
+      max-num-seqs: 64
+      max-num-batched-tokens: 256
       trust-remote-code: true
       no-enable-flashinfer-autotune: true
       block-size: 256
       compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
-      max-cudagraph-capture-size: 1024
-      gpu-memory-utilization: 0.92
+      max-cudagraph-capture-size: 256
+      gpu-memory-utilization: 0.90
       no-disable-hybrid-kv-cache-manager: true
       tokenizer-mode: "deepseek_v4"
       attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
-      speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
+      speculative-config: '{"method":"mtp","num_speculative_tokens":3}'
       moe-backend: "deep_gemm_amxf4_mega_moe"
       numa-bind: true
       numa-bind-nodes: [0, 0, 1, 1]
@@ -136,20 +139,21 @@ backend:
     VLLM_SERVER_DEV_MODE: "1"
     VLLM_USE_V2_MODEL_RUNNER: "1"
     VLLM_MOONCAKE_LOAD_RECV_THREADS: "20"
+    VLLM_MOONCAKE_STORE_SEND_THREADS: "8"
     VLLM_ALLREDUCE_USE_SYMM_MEM: "0"
     UCX_MEMTYPE_CACHE: "n"
     UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1"
     UCX_TLS: "rc,cuda_copy"
     NCCL_CUMEM_ENABLE: "1"
     NCCL_MNNVL_ENABLE: "1"
     NCCL_NVLS_ENABLE: "1"
-    NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
     VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
+    NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
     VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
     VLLM_USE_BREAKABLE_CUDAGRAPH: "0"
     VLLM_CONNECTOR_PREFETCH_DEPTH: "8"
     VLLM_DSV4_MEGA_FP8_COMBINE: "1"
-    DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}"
+    DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}"
     MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
     MC_STORE_CLIENT_METRIC: "1"
     MC_STORE_CLIENT_METRIC_INTERVAL: "5"
@@ -172,11 +176,11 @@ backend:
     NCCL_CUMEM_ENABLE: "1"
     NCCL_MNNVL_ENABLE: "1"
     NCCL_NVLS_ENABLE: "1"
-    NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
     VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
+    NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
     VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
     VLLM_DSV4_MEGA_FP8_COMBINE: "1"
-    DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}"
+    DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep12-decode-{job_id}"
     MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
     MC_STORE_CLIENT_METRIC: "1"
     MC_STORE_CLIENT_METRIC_INTERVAL: "5"
@@ -199,6 +203,8 @@ benchmark:
     IS_MULTINODE: "true"
     AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
     AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
+    # Avoid concurrent readers observing a mismatched mmap data/index pair.
+    AIPERF_DATASET_MMAP_CACHE_ENABLED: "false"
     AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
     HF_HUB_CACHE: "/hf_hub_cache"
     WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"