diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 26bde59696..e7bdf27542 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -2039,6 +2039,9 @@ build_replay_cmd() { # CPU on minimax-m2.5 at high concurrency. Lossless for vLLM (server # usage is authoritative). REPLAY_CMD+=" --use-server-token-count" + if [ -n "${AIPERF_EXTRA_INPUTS:-}" ]; then + REPLAY_CMD+=" --extra-inputs $AIPERF_EXTRA_INPUTS" + fi # Dynamo's KV router needs an explicit conversation session binding to # keep later turns on the prefill worker that owns their prefix blocks. # X-Correlation-ID is useful tracing metadata but does not establish that diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml new file mode 100644 index 0000000000..4686d003fc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml @@ -0,0 +1,80 @@ +name: "minimax-m3-vllm-agg-gb200-dep4-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } + args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: { aggregated: true } + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + vllm_config: + aggregated: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml new file mode 100644 index 0000000000..528e08b835 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml @@ -0,0 +1,82 @@ +name: "minimax-m3-vllm-agg-gb200-dep4-vllm-simple-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } + args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: { aggregated: true } + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + vllm_config: + aggregated: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml new file mode 100644 index 0000000000..b8b4c33411 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml @@ -0,0 +1,80 @@ +name: "minimax-m3-vllm-agg-gb200-dep8-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 2, agg_workers: 1, gpus_per_agg: 8 } +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } + args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: { aggregated: true } + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + vllm_config: + aggregated: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml new file mode 100644 index 0000000000..b74dd665b4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml @@ -0,0 +1,99 @@ +name: "minimax-m3-vllm-agg-gb200-tp4-agentic" + +model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:v0.27.1" + precision: "fp4" + +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } + +dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } + +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } + +frontend: + type: dynamo + enable_multiple_frontends: false + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } + args: + router-mode: "kv" + router-kv-events: true + router-reset-states: true + router-temperature: "0" + router-session-affinity-ttl-secs: 14400 + kv-cache-block-size: 128 + +backend: + type: vllm + connector: null + kv_events_config: { aggregated: true } + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + vllm_config: + aggregated: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml new file mode 100644 index 0000000000..1d62599edd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml @@ -0,0 +1,87 @@ +name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-agentic" + +model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:v0.27.1" + precision: "fp4" + +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } + +dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } + +frontend: + type: dynamo + enable_multiple_frontends: false + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } + args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 } + +backend: + type: vllm + connector: null + kv_events_config: { aggregated: true } + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + vllm_config: + aggregated: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + stream-interval: 20 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml new file mode 100644 index 0000000000..7c2343f404 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml @@ -0,0 +1,117 @@ +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep8-dep4-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 4 +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } + args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: true + prefill_environment: &worker_env + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: "300" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: *worker_env + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"PIECEWISE"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-cache-metrics: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b469673b45..8fd0430491 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7246,6 +7246,119 @@ minimaxm3-fp4-b200-vllm-agentic-mtp: search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 5, 8, 10, 12, 15, 20] } - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [20, 30, 40] } +# Preserve the B200 TP4 search space and add the GB200 Pareto candidates found +# by direct DEP and P/D tuning. +minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: + image: vllm/vllm-openai:v0.27.1 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.1" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.61 + search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 2, 5, 8, 10, 12, 15, 20] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 0, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [20, 30, 40] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 0, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [4, 32] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 0, tp: 4, ep: 4, dp-attn: true } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [32, 40] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 0, tp: 4, ep: 4, dp-attn: true } + - spec-decoding: mtp + kv-offloading: none + conc-list: [48] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 0, tp: 8, ep: 8, dp-attn: true } + +minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: + image: vllm/vllm-openai:v0.27.1 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.61 + search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [48] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 1, tp: 4, ep: 4, dp-attn: true } + dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 067b8a79ff..bc516a3357 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5981,3 +5981,13 @@ - "Inject the committed synthetic MTP acceptance length only for GB300 AgentX throughput; keep eval-only jobs on real target verification." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2571 + +- config-keys: + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Add the tuned GB200 MiniMax-M3 FP4 AgentX frontier with EAGLE3, the B200 TP4 baseline, DEP4/DEP8, SimpleCPU offload, and KV-routed P/D." + - "Use full-decode-only CUDA graphs for TP4 stability." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index ed4cbed78b..c4fb8899b3 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -251,8 +251,11 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/mnt/lustre01/models/MiniMax-M3-MXFP8" export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" + elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="/mnt/lustre01/models/MiniMax-M3-NVFP4" + export SRT_SLURM_MODEL_PREFIX="minimax-m3-nvfp4" else - echo "Unsupported model prefix/precision combination: $MODEL_PREFIX/$PRECISION. Supported combinations for dynamo-vllm: kimik2.5/fp4, kimik3/fp4, dsv4/fp4, minimaxm2.5/fp4, minimaxm2.5/fp8, minimaxm3/fp8" + echo "Unsupported model prefix/precision combination: $MODEL_PREFIX/$PRECISION. Supported combinations for dynamo-vllm: kimik2.5/fp4, kimik3/fp4, dsv4/fp4, minimaxm2.5/fp4, minimaxm2.5/fp8, minimaxm3/fp4, minimaxm3/fp8" exit 1 fi else @@ -391,9 +394,21 @@ if [ -d "$SRT_REPO_DIR" ]; then rm -rf "$SRT_REPO_DIR" fi +# MiniMax-M3 FP4 AgentX uses v1.0.50 for complete logical-worker metrics +# discovery across aggregate, DP-attention, and disaggregated topologies. +if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "minimaxm3" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" ]]; then + git clone --branch v1.0.50 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + test "$(git rev-parse HEAD)" = "e4019633c9e2bc25f38c44b81edf52bb0504d937" || { + echo "Error: NVIDIA/srt-slurm v1.0.50 resolved to an unexpected commit" >&2 + exit 1 + } + mkdir -p recipes/vllm/minimax-m3/gb200-fp4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic" \ + recipes/vllm/minimax-m3/gb200-fp4/agentic # These AgentX submissions use released srt-slurm custom-benchmark metrics # discovery so AIPerf receives every logical worker endpoint. -if [[ "$IS_AGENTIC" == "1" && (( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ) || ( "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" )) ]]; then +elif [[ "$IS_AGENTIC" == "1" && (( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ) || ( "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" )) ]]; then git clone --branch v1.0.45 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" test "$(git rev-parse HEAD)" = "9d8d92b20c350a5d42f0709f5a0b64e30eb37d33" || {