From b75fb7ed26b8ff4241e8f4c2a3e5840dd39ca8d0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 12:20:57 -0500 Subject: [PATCH 01/14] Add GB200 MiniMax M3 AgentX sweep --- .../gb200-fp4/agentic/agg-dep4-agentic.yaml | 74 +++++++++++ .../gb200-fp4/agentic/agg-dep8-agentic.yaml | 74 +++++++++++ .../gb200-fp4/agentic/agg-tp4-agentic.yaml | 93 ++++++++++++++ .../agentic/agg-tp4-vllm-simple-agentic.yaml | 80 ++++++++++++ .../disagg-1p1d-dep4-dep4-agentic.yaml | 87 +++++++++++++ .../disagg-1p1d-dep4-dep8-agentic.yaml | 110 ++++++++++++++++ .../disagg-2p1d-dep4-dep8-agentic.yaml | 110 ++++++++++++++++ configs/nvidia-master.yaml | 121 ++++++++++++++++++ perf-changelog.yaml | 10 ++ runners/launch_gb200-nv.sh | 17 ++- 10 files changed, 774 insertions(+), 2 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml new file mode 100644 index 0000000000..2d2d329415 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml @@ -0,0 +1,74 @@ +name: "minimax-m3-vllm-agg-gb200-dep4-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + frameworks: { dynamo: "1.3.0" } +dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: { aggregated: true } + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + vllm_config: + aggregated: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + tool-call-parser: "minimax_m3" + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml new file mode 100644 index 0000000000..d02dfa445c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml @@ -0,0 +1,74 @@ +name: "minimax-m3-vllm-agg-gb200-dep8-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + frameworks: { dynamo: "1.3.0" } +dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 2, agg_workers: 1, gpus_per_agg: 8 } +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: { aggregated: true } + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + vllm_config: + aggregated: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + tool-call-parser: "minimax_m3" + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml new file mode 100644 index 0000000000..28b491c91f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml @@ -0,0 +1,93 @@ +name: "minimax-m3-vllm-agg-gb200-tp4-agentic" + +model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + frameworks: { dynamo: "1.3.0" } + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-kv-events: true + router-temperature: "0" + kv-cache-block-size: 128 + +backend: + type: vllm + connector: null + kv_events_config: { aggregated: true } + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + vllm_config: + aggregated: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + tool-call-parser: "minimax_m3" + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml new file mode 100644 index 0000000000..b1cb45c9b3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml @@ -0,0 +1,80 @@ +name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-agentic" + +model: + path: "minimax-m3-nvfp4" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp4" + +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + frameworks: { dynamo: "1.3.0" } + +dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } + +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + +backend: + type: vllm + connector: null + kv_events_config: { aggregated: true } + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + vllm_config: + aggregated: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + trust-remote-code: true + enable-prefix-caching: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + tool-call-parser: "minimax_m3" + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml new file mode 100644 index 0000000000..54ab0e7d47 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml @@ -0,0 +1,87 @@ +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep4-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + frameworks: { dynamo: "1.3.0" } +dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: true + prefill_environment: &worker_env + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + decode_environment: *worker_env + vllm_config: + prefill: &dep4_config + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"PIECEWISE"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + tool-call-parser: "minimax_m3" + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + decode: + <<: *dep4_config + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml new file mode 100644 index 0000000000..6b994fa4c2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml @@ -0,0 +1,110 @@ +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep8-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + frameworks: { dynamo: "1.3.0" } +dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: true + prefill_environment: &worker_env + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + decode_environment: *worker_env + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"PIECEWISE"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + tool-call-parser: "minimax_m3" + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + tool-call-parser: "minimax_m3" + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml new file mode 100644 index 0000000000..6fa9ebfa25 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml @@ -0,0 +1,110 @@ +name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep8-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + frameworks: { dynamo: "1.3.0" } +dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: true + prefill_environment: &worker_env + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + decode_environment: *worker_env + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"PIECEWISE"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + tool-call-parser: "minimax_m3" + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.9 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + tool-call-parser: "minimax_m3" + enable-auto-tool-choice: true + default-chat-template-kwargs: '{"thinking_mode":"enabled"}' +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index caa0a7e082..0d6bea9faf 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7465,6 +7465,127 @@ minimaxm3-fp4-b200-vllm-agentic-mtp: search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 5, 8, 10, 12, 15, 20] } - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [20, 30, 40] } + +# GB200 MiniMax-M3 NVFP4 AgentX discovery grid. The aggregate TP4 arms retain +# the complete B200 search space; DEP and P/D add higher-concurrency probes. +minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: + image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.61 + search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 2, 5, 8, 10, 12, 15, 20] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 0, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [20, 30, 40] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 0, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: none + conc-list: [8, 12, 16, 20, 24, 32, 40, 48] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 0, tp: 4, ep: 4, dp-attn: true } + - spec-decoding: mtp + kv-offloading: none + conc-list: [32, 48, 64, 80, 96, 128] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 0, tp: 8, ep: 8, dp-attn: true } + +minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: + image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [16, 24, 32, 40, 48, 64] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 1, tp: 4, ep: 4, dp-attn: true } + - spec-decoding: mtp + conc-list: [48, 64, 80, 96, 128] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } + - spec-decoding: mtp + conc-list: [128, 160, 192, 256] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml" + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" + decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } dsv4-fp4-gb200-dynamo-vllm-agentic-3p2d-tep8-tp8: image: vllm/vllm-openai:v0.23.0 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fbe8a2ce2e..8d046caa12 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5960,3 +5960,13 @@ - "Use native EAGLE MTP (3 steps, top-k 1, 4 draft tokens) and golden synthetic acceptance length 2.49 for throughput; eval retains real verification." - "Follow the official SGLang DeepSeek-V4 Blackwell recipe, require nonempty SGLang server metrics, keep pooled AgentX connections alive, let AIPerf own HiCache warmup, and reserve transient MoE workspace at DEP8 c512." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2577 + +- config-keys: + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Add GB200 MiniMax-M3 NVFP4 AgentX with EAGLE3-GQA, the complete B200 TP4 baseline, vLLM SimpleCPU offload, DEP4/DEP8, and KV-routed P/D topologies." + - "Use NVIDIA/srt-slurm v1.0.50 and require every logical vLLM metrics endpoint." + pr-link: XXX diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 660ae078d5..71c1b89e2d 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -251,8 +251,11 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/mnt/lustre01/models/MiniMax-M3-MXFP8" export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" + elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="/mnt/lustre01/models/MiniMax-M3-NVFP4" + export SRT_SLURM_MODEL_PREFIX="minimax-m3-nvfp4" else - echo "Unsupported model prefix/precision combination: $MODEL_PREFIX/$PRECISION. Supported combinations for dynamo-vllm: kimik2.5/fp4, kimik3/fp4, dsv4/fp4, minimaxm2.5/fp4, minimaxm2.5/fp8, minimaxm3/fp8" + echo "Unsupported model prefix/precision combination: $MODEL_PREFIX/$PRECISION. Supported combinations for dynamo-vllm: kimik2.5/fp4, kimik3/fp4, dsv4/fp4, minimaxm2.5/fp4, minimaxm2.5/fp8, minimaxm3/fp4, minimaxm3/fp8" exit 1 fi else @@ -393,7 +396,17 @@ fi # Qwen3.5 FP4 AgentX uses the latest released srt-slurm. v1.0.45 injects # the aggregate logical-worker Prometheus endpoint into custom benchmarks. -if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then +if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "minimaxm3" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" ]]; then + git clone --branch v1.0.50 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + test "$(git rev-parse HEAD)" = "e4019633c9e2bc25f38c44b81edf52bb0504d937" || { + echo "Error: NVIDIA/srt-slurm v1.0.50 resolved to an unexpected commit" >&2 + exit 1 + } + mkdir -p recipes/vllm/minimax-m3/gb200-fp4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic" \ + recipes/vllm/minimax-m3/gb200-fp4/agentic +elif [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then git clone --branch v1.0.45 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" test "$(git rev-parse HEAD)" = "9d8d92b20c350a5d42f0709f5a0b64e30eb37d33" || { From 41d7d47e42c09213361644d84de95588c8e9fd88 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 12:21:38 -0500 Subject: [PATCH 02/14] Link MiniMax M3 performance entry --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8d046caa12..2be58ac6e8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5969,4 +5969,4 @@ description: - "Add GB200 MiniMax-M3 NVFP4 AgentX with EAGLE3-GQA, the complete B200 TP4 baseline, vLLM SimpleCPU offload, DEP4/DEP8, and KV-routed P/D topologies." - "Use NVIDIA/srt-slurm v1.0.50 and require every logical vLLM metrics endpoint." - pr-link: XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609 From 915d4a581b0d687288668db107d00606e4e38b99 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 12:26:18 -0500 Subject: [PATCH 03/14] Use stable vLLM for MiniMax M3 --- .../vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml | 4 ++-- .../vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml | 4 ++-- .../vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml | 4 ++-- .../gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml | 4 ++-- .../gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml | 4 ++-- .../gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml | 4 ++-- .../gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml index 2d2d329415..9a12083497 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml @@ -1,9 +1,9 @@ name: "minimax-m3-vllm-agg-gb200-dep4-agentic" -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7", precision: "fp4" } +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + container: { image: "vllm/vllm-openai:v0.27.1" } frameworks: { dynamo: "1.3.0" } dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } slurm: { time_limit: "12:00:00" } diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml index d02dfa445c..3eda8cf165 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml @@ -1,9 +1,9 @@ name: "minimax-m3-vllm-agg-gb200-dep8-agentic" -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7", precision: "fp4" } +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + container: { image: "vllm/vllm-openai:v0.27.1" } frameworks: { dynamo: "1.3.0" } dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } slurm: { time_limit: "12:00:00" } diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml index 28b491c91f..38814ac1b5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml @@ -2,12 +2,12 @@ name: "minimax-m3-vllm-agg-gb200-tp4-agentic" model: path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + container: "vllm/vllm-openai:v0.27.1" precision: "fp4" identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + container: { image: "vllm/vllm-openai:v0.27.1" } frameworks: { dynamo: "1.3.0" } dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml index b1cb45c9b3..fdaa93d3f6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml @@ -2,12 +2,12 @@ name: "minimax-m3-vllm-agg-gb200-tp4-vllm-simple-agentic" model: path: "minimax-m3-nvfp4" - container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + container: "vllm/vllm-openai:v0.27.1" precision: "fp4" identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + container: { image: "vllm/vllm-openai:v0.27.1" } frameworks: { dynamo: "1.3.0" } dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml index 54ab0e7d47..ac9456d2b1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml @@ -1,9 +1,9 @@ name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep4-agentic" -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7", precision: "fp4" } +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + container: { image: "vllm/vllm-openai:v0.27.1" } frameworks: { dynamo: "1.3.0" } dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } slurm: { time_limit: "12:00:00" } diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml index 6b994fa4c2..0aa48274ba 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml @@ -1,9 +1,9 @@ name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep8-agentic" -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7", precision: "fp4" } +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + container: { image: "vllm/vllm-openai:v0.27.1" } frameworks: { dynamo: "1.3.0" } dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } slurm: { time_limit: "12:00:00" } diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml index 6fa9ebfa25..d3ac8a7002 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml @@ -1,9 +1,9 @@ name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep8-agentic" -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7", precision: "fp4" } +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" } + container: { image: "vllm/vllm-openai:v0.27.1" } frameworks: { dynamo: "1.3.0" } dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } slurm: { time_limit: "12:00:00" } diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0d6bea9faf..87675962f2 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7395,7 +7395,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: decode: { num-worker: 0, tp: 2, ep: 2, dp-attn: false } minimaxm3-fp4-b300-vllm-agentic-mtp: - image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 + image: vllm/vllm-openai:v0.27.1 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:b300-nv @@ -7452,7 +7452,7 @@ minimaxm3-fp4-b300-vllm-agentic-mtp: # the same 3 TB AgentX ceiling before the proportional-GPU rule is applied. # GPU-resident points receive a zero budget. minimaxm3-fp4-b200-vllm-agentic-mtp: - image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 + image: vllm/vllm-openai:v0.27.1 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:b200-dgxc diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2be58ac6e8..5bc1c92ba8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5968,5 +5968,5 @@ - agentic-coding description: - "Add GB200 MiniMax-M3 NVFP4 AgentX with EAGLE3-GQA, the complete B200 TP4 baseline, vLLM SimpleCPU offload, DEP4/DEP8, and KV-routed P/D topologies." - - "Use NVIDIA/srt-slurm v1.0.50 and require every logical vLLM metrics endpoint." + - "Use vLLM v0.27.1 and NVIDIA/srt-slurm v1.0.50, and require every logical vLLM metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609 From 20df68754bf41a66794f3e8e800c9a7580dd2bef Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 12:29:18 -0500 Subject: [PATCH 04/14] Scope stable vLLM to GB200 MiniMax --- configs/nvidia-master.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 87675962f2..4609e7ed4d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7395,7 +7395,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: decode: { num-worker: 0, tp: 2, ep: 2, dp-attn: false } minimaxm3-fp4-b300-vllm-agentic-mtp: - image: vllm/vllm-openai:v0.27.1 + image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:b300-nv @@ -7452,7 +7452,7 @@ minimaxm3-fp4-b300-vllm-agentic-mtp: # the same 3 TB AgentX ceiling before the proportional-GPU rule is applied. # GPU-resident points receive a zero budget. minimaxm3-fp4-b200-vllm-agentic-mtp: - image: vllm/vllm-openai:v0.27.1 + image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:b200-dgxc @@ -7469,7 +7469,7 @@ minimaxm3-fp4-b200-vllm-agentic-mtp: # GB200 MiniMax-M3 NVFP4 AgentX discovery grid. The aggregate TP4 arms retain # the complete B200 search space; DEP and P/D add higher-concurrency probes. minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: - image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 + image: vllm/vllm-openai:v0.27.1 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:gb200-nv @@ -7537,7 +7537,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: decode: { num-worker: 0, tp: 8, ep: 8, dp-attn: true } minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: - image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 + image: vllm/vllm-openai:v0.27.1 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:gb200-nv From 8014a8439aceb37dbb985a65479a88fcae874b49 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 12:46:38 -0500 Subject: [PATCH 05/14] Align MiniMax routing with Dynamo 1.3 --- .../vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml | 2 +- .../vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml | 2 +- .../vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml | 1 - .../gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml | 2 +- .../gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml | 2 +- .../gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml | 2 +- .../gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml | 2 +- 7 files changed, 6 insertions(+), 7 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml index 9a12083497..c100f72efa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml @@ -13,7 +13,7 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false - args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } backend: type: vllm connector: null diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml index 3eda8cf165..5808b10545 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml @@ -13,7 +13,7 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false - args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } backend: type: vllm connector: null diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml index 38814ac1b5..5ab7e7d47d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml @@ -31,7 +31,6 @@ frontend: enable_multiple_frontends: false args: router-mode: "kv" - router-reset-states: true router-kv-events: true router-temperature: "0" kv-cache-block-size: 128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml index fdaa93d3f6..15f21253ca 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml @@ -19,7 +19,7 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false - args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } backend: type: vllm diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml index ac9456d2b1..e8d100569a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml @@ -21,7 +21,7 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false - args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } backend: type: vllm connector: null diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml index 0aa48274ba..7ce9937e6f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml @@ -21,7 +21,7 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false - args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } backend: type: vllm connector: null diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml index d3ac8a7002..01df089435 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml @@ -21,7 +21,7 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false - args: { router-mode: "kv", router-reset-states: true, router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } backend: type: vllm connector: null From 63f7a8c432d35235836a01b3c2946b5225a6e3d5 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 13:21:25 -0500 Subject: [PATCH 06/14] fix: use Dynamo MiniMax response parsers --- benchmarks/benchmark_lib.sh | 3 +++ .../gb200-fp4/agentic/agg-dep4-agentic.yaml | 6 +++--- .../gb200-fp4/agentic/agg-dep8-agentic.yaml | 6 +++--- .../minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml | 6 +++--- .../agentic/agg-tp4-vllm-simple-agentic.yaml | 6 +++--- .../agentic/disagg-1p1d-dep4-dep4-agentic.yaml | 6 +++--- .../agentic/disagg-1p1d-dep4-dep8-agentic.yaml | 11 +++++------ .../agentic/disagg-2p1d-dep4-dep8-agentic.yaml | 11 +++++------ perf-changelog.yaml | 2 +- 9 files changed, 29 insertions(+), 28 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 8cc894940f..752a154468 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -2038,6 +2038,9 @@ build_replay_cmd() { # CPU on minimax-m2.5 at high concurrency. Lossless for vLLM (server # usage is authoritative). REPLAY_CMD+=" --use-server-token-count" + if [ -n "${AIPERF_EXTRA_INPUTS:-}" ]; then + REPLAY_CMD+=" --extra-inputs $AIPERF_EXTRA_INPUTS" + fi # Dynamo's KV router needs an explicit conversation session binding to # keep later turns on the prefill worker that owns their prefix blocks. # X-Correlation-ID is useful tracing metadata but does not establish that diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml index c100f72efa..a2ff3f66f8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml @@ -52,9 +52,8 @@ backend: max-num-batched-tokens: 16384 no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" - tool-call-parser: "minimax_m3" - enable-auto-tool-choice: true - default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" sbatch_directives: { cpus-per-task: "144", mem: "0" } srun_options: { container-remap-root: "" } benchmark: @@ -68,6 +67,7 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml index 5808b10545..cc14fff423 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml @@ -52,9 +52,8 @@ backend: max-num-batched-tokens: 16384 no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" - tool-call-parser: "minimax_m3" - enable-auto-tool-choice: true - default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" sbatch_directives: { cpus-per-task: "144", mem: "0" } srun_options: { container-remap-root: "" } benchmark: @@ -68,6 +67,7 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml index 5ab7e7d47d..420dbf440d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml @@ -68,9 +68,8 @@ backend: max-num-batched-tokens: 16384 no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" - tool-call-parser: "minimax_m3" - enable-auto-tool-choice: true - default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" sbatch_directives: { cpus-per-task: "144", mem: "0" } srun_options: { container-remap-root: "" } @@ -86,6 +85,7 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml index 15f21253ca..6df4d01c86 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml @@ -56,9 +56,8 @@ backend: max-num-batched-tokens: 16384 no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" - tool-call-parser: "minimax_m3" - enable-auto-tool-choice: true - default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" sbatch_directives: { cpus-per-task: "144", mem: "0" } srun_options: { container-remap-root: "" } @@ -74,6 +73,7 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml index e8d100569a..22b09d61f5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml @@ -62,9 +62,8 @@ backend: max-num-batched-tokens: 16384 no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" - tool-call-parser: "minimax_m3" - enable-auto-tool-choice: true - default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" decode: <<: *dep4_config compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' @@ -81,6 +80,7 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml index 7ce9937e6f..6a58fadeda 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml @@ -62,9 +62,8 @@ backend: max-num-batched-tokens: 16384 no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" - tool-call-parser: "minimax_m3" - enable-auto-tool-choice: true - default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" decode: served-model-name: "nvidia/MiniMax-M3-NVFP4" tensor-parallel-size: 1 @@ -88,9 +87,8 @@ backend: max-num-batched-tokens: 16384 no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" - tool-call-parser: "minimax_m3" - enable-auto-tool-choice: true - default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" sbatch_directives: { cpus-per-task: "144", mem: "0" } srun_options: { container-remap-root: "" } benchmark: @@ -104,6 +102,7 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml index 01df089435..ea56a2fb57 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml @@ -62,9 +62,8 @@ backend: max-num-batched-tokens: 16384 no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" - tool-call-parser: "minimax_m3" - enable-auto-tool-choice: true - default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" decode: served-model-name: "nvidia/MiniMax-M3-NVFP4" tensor-parallel-size: 1 @@ -88,9 +87,8 @@ backend: max-num-batched-tokens: 16384 no-enable-flashinfer-autotune: true reasoning-parser: "minimax_m3" - tool-call-parser: "minimax_m3" - enable-auto-tool-choice: true - default-chat-template-kwargs: '{"thinking_mode":"enabled"}' + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" sbatch_directives: { cpus-per-task: "144", mem: "0" } srun_options: { container-remap-root: "" } benchmark: @@ -104,6 +102,7 @@ benchmark: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 40ff69a852..de6f6e35e7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5979,5 +5979,5 @@ - agentic-coding description: - "Add GB200 MiniMax-M3 NVFP4 AgentX with EAGLE3-GQA, the complete B200 TP4 baseline, vLLM SimpleCPU offload, DEP4/DEP8, and KV-routed P/D topologies." - - "Use vLLM v0.27.1 and NVIDIA/srt-slurm v1.0.50, and require every logical vLLM metrics endpoint." + - "Use vLLM v0.27.1, NVIDIA/srt-slurm v1.0.50, Dynamo-native MiniMax tool/reasoning parsing, thinking-enabled requests, and every logical vLLM metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609 From 3837e71425d058dc8028ef363a3b1078f5f4692b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 13:55:07 -0500 Subject: [PATCH 07/14] fix: reserve MiniMax DEP KV capacity --- .../vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml | 2 +- .../vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml | 2 +- .../gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml | 2 +- .../gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml | 4 ++-- .../gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml | 4 ++-- perf-changelog.yaml | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml index a2ff3f66f8..71b3f5c452 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml @@ -41,7 +41,7 @@ backend: enable-prefix-caching: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 - gpu-memory-utilization: 0.9 + gpu-memory-utilization: 0.95 max-model-len: 1048576 language-model-only: true kv-cache-dtype: "fp8" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml index cc14fff423..8cb17b10e0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml @@ -41,7 +41,7 @@ backend: enable-prefix-caching: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 - gpu-memory-utilization: 0.9 + gpu-memory-utilization: 0.95 max-model-len: 1048576 language-model-only: true kv-cache-dtype: "fp8" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml index 22b09d61f5..5f9209217a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml @@ -51,7 +51,7 @@ backend: kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 - gpu-memory-utilization: 0.9 + gpu-memory-utilization: 0.95 max-model-len: 1048576 language-model-only: true kv-cache-dtype: "fp8" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml index 6a58fadeda..f5aad7c1c6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml @@ -51,7 +51,7 @@ backend: kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 - gpu-memory-utilization: 0.9 + gpu-memory-utilization: 0.95 max-model-len: 1048576 language-model-only: true kv-cache-dtype: "fp8" @@ -76,7 +76,7 @@ backend: kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 - gpu-memory-utilization: 0.9 + gpu-memory-utilization: 0.95 max-model-len: 1048576 language-model-only: true kv-cache-dtype: "fp8" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml index ea56a2fb57..b38e4883f1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml @@ -51,7 +51,7 @@ backend: kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 - gpu-memory-utilization: 0.9 + gpu-memory-utilization: 0.95 max-model-len: 1048576 language-model-only: true kv-cache-dtype: "fp8" @@ -76,7 +76,7 @@ backend: kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 - gpu-memory-utilization: 0.9 + gpu-memory-utilization: 0.95 max-model-len: 1048576 language-model-only: true kv-cache-dtype: "fp8" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index de6f6e35e7..8cff438436 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5979,5 +5979,5 @@ - agentic-coding description: - "Add GB200 MiniMax-M3 NVFP4 AgentX with EAGLE3-GQA, the complete B200 TP4 baseline, vLLM SimpleCPU offload, DEP4/DEP8, and KV-routed P/D topologies." - - "Use vLLM v0.27.1, NVIDIA/srt-slurm v1.0.50, Dynamo-native MiniMax tool/reasoning parsing, thinking-enabled requests, and every logical vLLM metrics endpoint." + - "Use vLLM v0.27.1, NVIDIA/srt-slurm v1.0.50, Dynamo-native MiniMax tool/reasoning parsing, thinking-enabled requests, 0.95 DEP memory utilization for the full 1M context, and every logical vLLM metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609 From 2e8502a7d17cda601ab451ba4bea1b03333157e7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 14:12:33 -0500 Subject: [PATCH 08/14] fix: expose MiniMax prompt cache tokens --- .../vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml | 2 +- .../vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml | 2 +- .../vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml | 2 +- .../gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml | 2 +- .../gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml | 2 +- .../gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml | 3 ++- .../gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml | 3 ++- perf-changelog.yaml | 2 +- 8 files changed, 10 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml index 71b3f5c452..9cab12be19 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml @@ -23,7 +23,6 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "7200" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" VLLM_LOG_STATS_INTERVAL: "1" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" @@ -39,6 +38,7 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true + enable-prompt-tokens-details: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 gpu-memory-utilization: 0.95 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml index 8cb17b10e0..655deb5682 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml @@ -23,7 +23,6 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "7200" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" VLLM_LOG_STATS_INTERVAL: "1" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" @@ -39,6 +38,7 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true + enable-prompt-tokens-details: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 gpu-memory-utilization: 0.95 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml index 420dbf440d..8a3b780551 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml @@ -43,7 +43,6 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "7200" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" VLLM_LOG_STATS_INTERVAL: "1" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" @@ -56,6 +55,7 @@ backend: pipeline-parallel-size: 1 trust-remote-code: true enable-prefix-caching: true + enable-prompt-tokens-details: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 gpu-memory-utilization: 0.9 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml index 6df4d01c86..c48aa482e4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml @@ -29,7 +29,6 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "7200" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" VLLM_LOG_STATS_INTERVAL: "1" VLLM_USE_SIMPLE_KV_OFFLOAD: "1" NCCL_CUMEM_ENABLE: "1" @@ -43,6 +42,7 @@ backend: pipeline-parallel-size: 1 trust-remote-code: true enable-prefix-caching: true + enable-prompt-tokens-details: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 gpu-memory-utilization: 0.9 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml index 5f9209217a..5c2843eefa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml @@ -31,7 +31,6 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "7200" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" VLLM_LOG_STATS_INTERVAL: "1" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" @@ -48,6 +47,7 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true + enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml index f5aad7c1c6..3e4d02d989 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml @@ -31,7 +31,6 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "7200" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" VLLM_LOG_STATS_INTERVAL: "1" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" @@ -48,6 +47,7 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true + enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 @@ -73,6 +73,7 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true + enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml index b38e4883f1..542eb47117 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml @@ -31,7 +31,6 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "7200" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_MINIMAX_M3_MSA_DECODE_BACKEND: "cutlass" VLLM_LOG_STATS_INTERVAL: "1" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" @@ -48,6 +47,7 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true + enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 @@ -73,6 +73,7 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true + enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8cff438436..b80e824b1a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5979,5 +5979,5 @@ - agentic-coding description: - "Add GB200 MiniMax-M3 NVFP4 AgentX with EAGLE3-GQA, the complete B200 TP4 baseline, vLLM SimpleCPU offload, DEP4/DEP8, and KV-routed P/D topologies." - - "Use vLLM v0.27.1, NVIDIA/srt-slurm v1.0.50, Dynamo-native MiniMax tool/reasoning parsing, thinking-enabled requests, 0.95 DEP memory utilization for the full 1M context, and every logical vLLM metrics endpoint." + - "Use vLLM v0.27.1, NVIDIA/srt-slurm v1.0.50, Dynamo-native MiniMax tool/reasoning parsing, thinking-enabled requests, 0.95 DEP memory utilization for the full 1M context, prompt-cache token details, and every logical vLLM metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609 From 15d868e822182c5d781eb7433bad96fc0d57c105 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 14:55:31 -0500 Subject: [PATCH 09/14] Use stable Dynamo and add MiniMax offload probes --- .../gb200-fp4/agentic/agg-dep4-agentic.yaml | 4 +- .../agentic/agg-dep4-vllm-simple-agentic.yaml | 76 ++++++++++++ .../gb200-fp4/agentic/agg-dep8-agentic.yaml | 4 +- .../agentic/agg-dep8-vllm-simple-agentic.yaml | 76 ++++++++++++ .../gb200-fp4/agentic/agg-tp4-agentic.yaml | 6 +- .../agentic/agg-tp4-vllm-simple-agentic.yaml | 4 +- .../disagg-1p1d-dep4-dep4-agentic.yaml | 4 +- .../disagg-1p1d-dep4-dep8-agentic.yaml | 4 +- .../disagg-2p1d-dep4-dep8-agentic.yaml | 4 +- ...gg-2p1d-dep4-dep8-vllm-simple-agentic.yaml | 111 ++++++++++++++++++ .../disagg-3p1d-dep4-dep4-agentic.yaml | 87 ++++++++++++++ ...gg-3p1d-dep4-dep4-vllm-simple-agentic.yaml | 88 ++++++++++++++ configs/nvidia-master.yaml | 4 +- perf-changelog.yaml | 4 +- 14 files changed, 456 insertions(+), 20 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml index 9cab12be19..edac79caa3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml @@ -4,8 +4,8 @@ model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precis identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.0" } -dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml new file mode 100644 index 0000000000..812ad0cb0c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml @@ -0,0 +1,76 @@ +name: "minimax-m3-vllm-agg-gb200-dep4-vllm-simple-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: { aggregated: true } + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + vllm_config: + aggregated: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml index 655deb5682..cdea96a9a6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml @@ -4,8 +4,8 @@ model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precis identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.0" } -dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 2, agg_workers: 1, gpus_per_agg: 8 } diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml new file mode 100644 index 0000000000..cb8bad639d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml @@ -0,0 +1,76 @@ +name: "minimax-m3-vllm-agg-gb200-dep8-vllm-simple-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 2, agg_workers: 1, gpus_per_agg: 8 } +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: { aggregated: true } + aggregated_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + vllm_config: + aggregated: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml index 8a3b780551..01818dd65e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml @@ -8,11 +8,9 @@ model: identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.0" } + frameworks: { dynamo: "1.3.1" } -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true +dynamo: { version: "1.3.1", install: true } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml index c48aa482e4..b682c7ac7c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml @@ -8,9 +8,9 @@ model: identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.0" } + frameworks: { dynamo: "1.3.1" } -dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } +dynamo: { version: "1.3.1", install: true } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml index 5c2843eefa..f2a2bccaf5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml @@ -4,8 +4,8 @@ model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precis identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.0" } -dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml index 3e4d02d989..6a13d35c56 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml @@ -4,8 +4,8 @@ model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precis identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.0" } -dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml index 542eb47117..aabb05ed20 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml @@ -4,8 +4,8 @@ model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precis identity: model: { repo: "nvidia/MiniMax-M3-NVFP4" } container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.0" } -dynamo: { hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1", install: true } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml new file mode 100644 index 0000000000..df367dc1d8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml @@ -0,0 +1,111 @@ +name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep8-vllm-simple-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: true + prefill_environment: &worker_env + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + decode_environment: *worker_env + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"PIECEWISE"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml new file mode 100644 index 0000000000..9c9a2a1ac7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml @@ -0,0 +1,87 @@ +name: "minimax-m3-vllm-disagg-gb200-3p1d-dep4-dep4-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 3 + decode_nodes: 1 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: true + prefill_environment: &worker_env + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + decode_environment: *worker_env + vllm_config: + prefill: &vllm_role + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"PIECEWISE"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: + <<: *vllm_role + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml new file mode 100644 index 0000000000..6f3cc8e0ec --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml @@ -0,0 +1,88 @@ +name: "minimax-m3-vllm-disagg-gb200-3p1d-dep4-dep4-vllm-simple-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 3 + decode_nodes: 1 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: true + prefill_environment: &worker_env + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + decode_environment: *worker_env + vllm_config: + prefill: &vllm_role + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"PIECEWISE"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: + <<: *vllm_role + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0b9299a57d..8f78ae5f6e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7475,7 +7475,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: runner: cluster:gb200-nv precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0" } + router: { name: dynamo-router, version: "1.3.1" } multinode: true disagg: false scenarios: @@ -7543,7 +7543,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: runner: cluster:gb200-nv precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0" } + router: { name: dynamo-router, version: "1.3.1" } kv-p2p-transfer: nixl multinode: true disagg: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b80e824b1a..463d8311fd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5978,6 +5978,6 @@ scenario-type: - agentic-coding description: - - "Add GB200 MiniMax-M3 NVFP4 AgentX with EAGLE3-GQA, the complete B200 TP4 baseline, vLLM SimpleCPU offload, DEP4/DEP8, and KV-routed P/D topologies." - - "Use vLLM v0.27.1, NVIDIA/srt-slurm v1.0.50, Dynamo-native MiniMax tool/reasoning parsing, thinking-enabled requests, 0.95 DEP memory utilization for the full 1M context, prompt-cache token details, and every logical vLLM metrics endpoint." + - "Add GB200 MiniMax-M3 NVFP4 AgentX with EAGLE3-GQA, the B200 TP4 baseline, DEP4/DEP8, KV-routed P/D, and high-concurrency SimpleCPU offload." + - "Use vLLM v0.27.1, NVIDIA/srt-slurm v1.0.50, and Dynamo v1.3.1 with full 1M context, MiniMax parsing, prompt-cache token details, and every logical vLLM metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609 From a37912d8edda83b5b149a429457d914dfa83fdb1 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 15:03:25 -0500 Subject: [PATCH 10/14] Add MiniMax 3P1D DEP8 probes --- .../disagg-3p1d-dep4-dep8-agentic.yaml | 110 +++++++++++++++++ ...gg-3p1d-dep4-dep8-vllm-simple-agentic.yaml | 111 ++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml new file mode 100644 index 0000000000..1f50ef49b6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml @@ -0,0 +1,110 @@ +name: "minimax-m3-vllm-disagg-gb200-3p1d-dep4-dep8-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 3 + decode_nodes: 2 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: true + prefill_environment: &worker_env + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + decode_environment: *worker_env + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"PIECEWISE"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml new file mode 100644 index 0000000000..e566dc4172 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml @@ -0,0 +1,111 @@ +name: "minimax-m3-vllm-disagg-gb200-3p1d-dep4-dep8-vllm-simple-agentic" + +model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } +identity: + model: { repo: "nvidia/MiniMax-M3-NVFP4" } + container: { image: "vllm/vllm-openai:v0.27.1" } + frameworks: { dynamo: "1.3.1" } +dynamo: { version: "1.3.1", install: true } +slurm: { time_limit: "12:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 3 + decode_nodes: 2 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } +frontend: + type: dynamo + enable_multiple_frontends: false + args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } +backend: + type: vllm + connector: null + dp_launch_mode: per_node + kv_events_config: true + prefill_environment: &worker_env + VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_USE_SIMPLE_KV_OFFLOAD: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_TLS: "cuda_copy,cuda_ipc,rc" + decode_environment: *worker_env + vllm_config: + prefill: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"PIECEWISE"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" + decode: + served-model-name: "nvidia/MiniMax-M3-NVFP4" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + enable-prefix-caching: true + enable-prompt-tokens-details: true + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + block-size: 128 + gpu-memory-utilization: 0.95 + max-model-len: 1048576 + language-model-only: true + kv-cache-dtype: "fp8" + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + stream-interval: 20 + max-cudagraph-capture-size: 512 + max-num-batched-tokens: 16384 + no-enable-flashinfer-autotune: true + reasoning-parser: "minimax_m3" + dyn-tool-call-parser: "minimax_m3" + dyn-reasoning-parser: "minimax_m3" +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_EXTRA_INPUTS: "thinking:true" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" From e3281ae587d1e025650ca67e3ce937b088152e2a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 15:14:06 -0500 Subject: [PATCH 11/14] Remove unsupported MiniMax prompt-details flag --- .../vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml | 1 - .../gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml | 1 - .../vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml | 1 - .../gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml | 1 - .../vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml | 1 - .../gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml | 1 - .../gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml | 1 - .../gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml | 2 -- .../gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml | 2 -- .../agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml | 2 -- .../gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml | 1 - .../agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml | 1 - .../gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml | 2 -- .../agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml | 2 -- perf-changelog.yaml | 2 +- 15 files changed, 1 insertion(+), 20 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml index edac79caa3..e489bb798e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml @@ -38,7 +38,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 gpu-memory-utilization: 0.95 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml index 812ad0cb0c..77ae8c9237 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml @@ -39,7 +39,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 gpu-memory-utilization: 0.95 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml index cdea96a9a6..3850686a68 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml @@ -38,7 +38,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 gpu-memory-utilization: 0.95 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml index cb8bad639d..69b73ffda0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml @@ -39,7 +39,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 gpu-memory-utilization: 0.95 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml index 01818dd65e..0e9cd32e74 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml @@ -53,7 +53,6 @@ backend: pipeline-parallel-size: 1 trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 gpu-memory-utilization: 0.9 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml index b682c7ac7c..29de0ae85c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml @@ -42,7 +42,6 @@ backend: pipeline-parallel-size: 1 trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 gpu-memory-utilization: 0.9 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml index f2a2bccaf5..bdaec41088 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml @@ -47,7 +47,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml index 6a13d35c56..ca06842e5d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml @@ -47,7 +47,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 @@ -73,7 +72,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml index aabb05ed20..0033cd4cc3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml @@ -47,7 +47,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 @@ -73,7 +72,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml index df367dc1d8..ec7db0f77e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml @@ -48,7 +48,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 @@ -74,7 +73,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml index 9c9a2a1ac7..67e997ef3e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml @@ -47,7 +47,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml index 6f3cc8e0ec..3c6d4f3792 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml @@ -48,7 +48,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml index 1f50ef49b6..b1c43f47f4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml @@ -47,7 +47,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 @@ -73,7 +72,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml index e566dc4172..b2f46a9b43 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml @@ -48,7 +48,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 @@ -74,7 +73,6 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - enable-prompt-tokens-details: true kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' block-size: 128 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 463d8311fd..44b891e563 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5979,5 +5979,5 @@ - agentic-coding description: - "Add GB200 MiniMax-M3 NVFP4 AgentX with EAGLE3-GQA, the B200 TP4 baseline, DEP4/DEP8, KV-routed P/D, and high-concurrency SimpleCPU offload." - - "Use vLLM v0.27.1, NVIDIA/srt-slurm v1.0.50, and Dynamo v1.3.1 with full 1M context, MiniMax parsing, prompt-cache token details, and every logical vLLM metrics endpoint." + - "Use vLLM v0.27.1, NVIDIA/srt-slurm v1.0.50, and Dynamo v1.3.1 with full 1M context, MiniMax parsing, KV-cache token metrics, and every logical vLLM metrics endpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609 From d707e114a16cd7e71df05cc4e05bcd919455683e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 14 Aug 2026 21:52:23 -0500 Subject: [PATCH 12/14] perf(agentx): prune GB200 MiniMax tuning grid --- .../gb200-fp4/agentic/agg-dep4-agentic.yaml | 11 +- .../agentic/agg-dep4-vllm-simple-agentic.yaml | 11 +- .../gb200-fp4/agentic/agg-dep8-agentic.yaml | 11 +- .../agentic/agg-dep8-vllm-simple-agentic.yaml | 75 ------------ .../gb200-fp4/agentic/agg-tp4-agentic.yaml | 11 +- .../agentic/agg-tp4-vllm-simple-agentic.yaml | 11 +- .../disagg-1p1d-dep4-dep4-agentic.yaml | 86 -------------- ...aml => disagg-1p1d-dep8-dep4-agentic.yaml} | 29 +++-- .../disagg-2p1d-dep4-dep8-agentic.yaml | 108 ----------------- ...gg-2p1d-dep4-dep8-vllm-simple-agentic.yaml | 109 ------------------ .../disagg-3p1d-dep4-dep4-agentic.yaml | 86 -------------- ...gg-3p1d-dep4-dep4-vllm-simple-agentic.yaml | 87 -------------- .../disagg-3p1d-dep4-dep8-agentic.yaml | 108 ----------------- ...gg-3p1d-dep4-dep8-vllm-simple-agentic.yaml | 109 ------------------ configs/nvidia-master.yaml | 99 ++-------------- perf-changelog.yaml | 3 +- 16 files changed, 75 insertions(+), 879 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml rename benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/{disagg-1p1d-dep4-dep8-agentic.yaml => disagg-1p1d-dep8-dep4-agentic.yaml} (82%) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml index e489bb798e..4686d003fc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-agentic.yaml @@ -6,6 +6,7 @@ identity: container: { image: "vllm/vllm-openai:v0.27.1" } frameworks: { dynamo: "1.3.1" } dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } @@ -13,7 +14,8 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } + args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 } backend: type: vllm connector: null @@ -21,13 +23,17 @@ backend: kv_events_config: { aggregated: true } aggregated_environment: VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" VLLM_LOG_STATS_INTERVAL: "1" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" vllm_config: aggregated: served-model-name: "nvidia/MiniMax-M3-NVFP4" @@ -38,7 +44,8 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' block-size: 128 gpu-memory-utilization: 0.95 max-model-len: 1048576 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml index 77ae8c9237..528e08b835 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep4-vllm-simple-agentic.yaml @@ -6,6 +6,7 @@ identity: container: { image: "vllm/vllm-openai:v0.27.1" } frameworks: { dynamo: "1.3.1" } dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } @@ -13,7 +14,8 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } + args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 } backend: type: vllm connector: null @@ -21,6 +23,7 @@ backend: kv_events_config: { aggregated: true } aggregated_environment: VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" VLLM_LOG_STATS_INTERVAL: "1" @@ -28,7 +31,10 @@ backend: NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" vllm_config: aggregated: served-model-name: "nvidia/MiniMax-M3-NVFP4" @@ -39,7 +45,8 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' block-size: 128 gpu-memory-utilization: 0.95 max-model-len: 1048576 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml index 3850686a68..b8b4c33411 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-agentic.yaml @@ -6,6 +6,7 @@ identity: container: { image: "vllm/vllm-openai:v0.27.1" } frameworks: { dynamo: "1.3.1" } dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 2, agg_workers: 1, gpus_per_agg: 8 } @@ -13,7 +14,8 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } + args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 } backend: type: vllm connector: null @@ -21,13 +23,17 @@ backend: kv_events_config: { aggregated: true } aggregated_environment: VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" VLLM_LOG_STATS_INTERVAL: "1" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" vllm_config: aggregated: served-model-name: "nvidia/MiniMax-M3-NVFP4" @@ -38,7 +44,8 @@ backend: enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' block-size: 128 gpu-memory-utilization: 0.95 max-model-len: 1048576 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml deleted file mode 100644 index 69b73ffda0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml +++ /dev/null @@ -1,75 +0,0 @@ -name: "minimax-m3-vllm-agg-gb200-dep8-vllm-simple-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 2, agg_workers: 1, gpus_per_agg: 8 } -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: { aggregated: true } - aggregated_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_SIMPLE_KV_OFFLOAD: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - vllm_config: - aggregated: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml index 0e9cd32e74..aea6207c14 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml @@ -11,6 +11,7 @@ identity: frameworks: { dynamo: "1.3.1" } dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } @@ -27,10 +28,13 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } args: router-mode: "kv" router-kv-events: true + router-reset-states: true router-temperature: "0" + router-session-affinity-ttl-secs: 14400 kv-cache-block-size: 128 backend: @@ -39,13 +43,17 @@ backend: kv_events_config: { aggregated: true } aggregated_environment: VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" VLLM_LOG_STATS_INTERVAL: "1" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" vllm_config: aggregated: served-model-name: "nvidia/MiniMax-M3-NVFP4" @@ -53,7 +61,8 @@ backend: pipeline-parallel-size: 1 trust-remote-code: true enable-prefix-caching: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' block-size: 128 gpu-memory-utilization: 0.9 max-model-len: 1048576 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml index 29de0ae85c..247690d2ab 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml @@ -11,6 +11,7 @@ identity: frameworks: { dynamo: "1.3.1" } dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: { gpu_type: "gb200", gpus_per_node: 4, agg_nodes: 1, agg_workers: 1, gpus_per_agg: 4 } @@ -19,7 +20,8 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } + args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 } backend: type: vllm @@ -27,6 +29,7 @@ backend: kv_events_config: { aggregated: true } aggregated_environment: VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" VLLM_LOG_STATS_INTERVAL: "1" @@ -34,7 +37,10 @@ backend: NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" vllm_config: aggregated: served-model-name: "nvidia/MiniMax-M3-NVFP4" @@ -42,7 +48,8 @@ backend: pipeline-parallel-size: 1 trust-remote-code: true enable-prefix-caching: true - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + kv-cache-metrics: true + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' block-size: 128 gpu-memory-utilization: 0.9 max-model-len: 1048576 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml deleted file mode 100644 index bdaec41088..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep4-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: true - prefill_environment: &worker_env - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - decode_environment: *worker_env - vllm_config: - prefill: &dep4_config - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - <<: *dep4_config - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml similarity index 82% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml index ca06842e5d..7c2343f404 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml @@ -1,4 +1,4 @@ -name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep8-agentic" +name: "minimax-m3-vllm-disagg-gb200-1p1d-dep8-dep4-agentic" model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } identity: @@ -6,22 +6,24 @@ identity: container: { image: "vllm/vllm-openai:v0.27.1" } frameworks: { dynamo: "1.3.1" } dynamo: { version: "1.3.1", install: true } +environment: { ETCD_LEASE_TTL: "7200" } slurm: { time_limit: "12:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } resources: gpu_type: "gb200" gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 + prefill_nodes: 2 + decode_nodes: 1 prefill_workers: 1 decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 + gpus_per_prefill: 8 + gpus_per_decode: 4 infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } + env: { DYN_TCP_REQUEST_TIMEOUT: "60" } + args: { router-mode: "kv", router-kv-events: true, router-reset-states: true, router-temperature: "0", router-session-affinity-ttl-secs: 14400, kv-cache-block-size: 128 } backend: type: vllm connector: null @@ -29,26 +31,32 @@ backend: kv_events_config: true prefill_environment: &worker_env VLLM_ENGINE_READY_TIMEOUT_S: "7200" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: "300" VLLM_FLOAT32_MATMUL_PRECISION: "high" VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" VLLM_LOG_STATS_INTERVAL: "1" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" UCX_TLS: "cuda_copy,cuda_ipc,rc" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" decode_environment: *worker_env vllm_config: prefill: served-model-name: "nvidia/MiniMax-M3-NVFP4" tensor-parallel-size: 1 pipeline-parallel-size: 1 - data-parallel-size: 4 + data-parallel-size: 8 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true + kv-cache-metrics: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' block-size: 128 gpu-memory-utilization: 0.95 max-model-len: 1048576 @@ -67,13 +75,14 @@ backend: served-model-name: "nvidia/MiniMax-M3-NVFP4" tensor-parallel-size: 1 pipeline-parallel-size: 1 - data-parallel-size: 8 + data-parallel-size: 4 data-parallel-rpc-port: 13345 enable-expert-parallel: true trust-remote-code: true enable-prefix-caching: true + kv-cache-metrics: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' + attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' block-size: 128 gpu-memory-utilization: 0.95 max-model-len: 1048576 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml deleted file mode 100644 index 0033cd4cc3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml +++ /dev/null @@ -1,108 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep8-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: true - prefill_environment: &worker_env - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - decode_environment: *worker_env - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml deleted file mode 100644 index ec7db0f77e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml +++ /dev/null @@ -1,109 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-2p1d-dep4-dep8-vllm-simple-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: true - prefill_environment: &worker_env - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_SIMPLE_KV_OFFLOAD: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - decode_environment: *worker_env - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml deleted file mode 100644 index 67e997ef3e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-agentic.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-3p1d-dep4-dep4-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 1 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: true - prefill_environment: &worker_env - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - decode_environment: *worker_env - vllm_config: - prefill: &vllm_role - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - <<: *vllm_role - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml deleted file mode 100644 index 3c6d4f3792..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep4-vllm-simple-agentic.yaml +++ /dev/null @@ -1,87 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-3p1d-dep4-dep4-vllm-simple-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 1 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: true - prefill_environment: &worker_env - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_SIMPLE_KV_OFFLOAD: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - decode_environment: *worker_env - vllm_config: - prefill: &vllm_role - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - <<: *vllm_role - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml deleted file mode 100644 index b1c43f47f4..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml +++ /dev/null @@ -1,108 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-3p1d-dep4-dep8-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: true - prefill_environment: &worker_env - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - decode_environment: *worker_env - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml deleted file mode 100644 index b2f46a9b43..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml +++ /dev/null @@ -1,109 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-3p1d-dep4-dep8-vllm-simple-agentic" - -model: { path: "minimax-m3-nvfp4", container: "vllm/vllm-openai:v0.27.1", precision: "fp4" } -identity: - model: { repo: "nvidia/MiniMax-M3-NVFP4" } - container: { image: "vllm/vllm-openai:v0.27.1" } - frameworks: { dynamo: "1.3.1" } -dynamo: { version: "1.3.1", install: true } -slurm: { time_limit: "12:00:00" } -health_check: { max_attempts: 2160, interval_seconds: 10 } -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 -infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } -frontend: - type: dynamo - enable_multiple_frontends: false - args: { router-mode: "kv", router-kv-events: true, router-temperature: "0", kv-cache-block-size: 128 } -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: true - prefill_environment: &worker_env - VLLM_ENGINE_READY_TIMEOUT_S: "7200" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "trtllm" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_SIMPLE_KV_OFFLOAD: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - UCX_TLS: "cuda_copy,cuda_ipc,rc" - decode_environment: *worker_env - vllm_config: - prefill: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" - decode: - served-model-name: "nvidia/MiniMax-M3-NVFP4" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - enable-prefix-caching: true - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both"},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}]}}' - attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - block-size: 128 - gpu-memory-utilization: 0.95 - max-model-len: 1048576 - language-model-only: true - kv-cache-dtype: "fp8" - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - stream-interval: 20 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 - no-enable-flashinfer-autotune: true - reasoning-parser: "minimax_m3" - dyn-tool-call-parser: "minimax_m3" - dyn-reasoning-parser: "minimax_m3" -sbatch_directives: { cpus-per-task: "144", mem: "0" } -srun_options: { container-remap-root: "" } -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_EXTRA_INPUTS: "thinking:true" - AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7fc8647105..8fd0430491 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7246,8 +7246,8 @@ minimaxm3-fp4-b200-vllm-agentic-mtp: search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 5, 8, 10, 12, 15, 20] } - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [20, 30, 40] } -# GB200 MiniMax-M3 NVFP4 AgentX discovery grid. The aggregate TP4 arms retain -# the complete B200 search space; DEP and P/D add higher-concurrency probes. +# Preserve the B200 TP4 search space and add the GB200 Pareto candidates found +# by direct DEP and P/D tuning. minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: image: vllm/vllm-openai:v0.27.1 model: nvidia/MiniMax-M3-NVFP4 @@ -7291,7 +7291,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: decode: { num-worker: 0, tp: 4, ep: 1, dp-attn: false } - spec-decoding: mtp kv-offloading: none - conc-list: [8, 12, 16, 20, 24, 32, 40, 48] + conc-list: [4, 32] prefill: num-worker: 1 tp: 4 @@ -7305,7 +7305,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple } - conc-list: [20, 24, 32, 40, 48] + conc-list: [32, 40] prefill: num-worker: 1 tp: 4 @@ -7318,7 +7318,7 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: decode: { num-worker: 0, tp: 4, ep: 4, dp-attn: true } - spec-decoding: mtp kv-offloading: none - conc-list: [32, 48, 64, 80, 96, 128] + conc-list: [48] prefill: num-worker: 1 tp: 8 @@ -7329,20 +7329,6 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" decode: { num-worker: 0, tp: 8, ep: 8, dp-attn: true } - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: vllm-simple } - conc-list: [48, 64, 80, 96, 128] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-dep8-vllm-simple-agentic.yaml" - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - decode: { num-worker: 0, tp: 8, ep: 8, dp-attn: true } minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: image: vllm/vllm-openai:v0.27.1 @@ -7361,84 +7347,17 @@ minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp: search-space: - spec-decoding: mtp kv-offloading: none - conc-list: [16, 24, 32, 40, 48, 64] + conc-list: [48] prefill: num-worker: 1 - tp: 4 - ep: 4 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep4-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep8-dep4-agentic.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" decode: { num-worker: 1, tp: 4, ep: 4, dp-attn: true } - - spec-decoding: mtp - kv-offloading: none - conc-list: [48, 64, 80, 96, 128] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-1p1d-dep4-dep8-agentic.yaml" - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } - - spec-decoding: mtp - kv-offloading: none - conc-list: [128, 160, 192, 256] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-agentic.yaml" - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: vllm-simple } - conc-list: [128, 160, 192, 256] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-2p1d-dep4-dep8-vllm-simple-agentic.yaml" - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } - - spec-decoding: mtp - kv-offloading: none - conc-list: [160, 192, 256] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-agentic.yaml" - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } - - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: vllm-simple } - conc-list: [160, 192, 256] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/gb200-fp4/agentic/disagg-3p1d-dep4-dep8-vllm-simple-agentic.yaml" - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.78" - decode: { num-worker: 1, tp: 8, ep: 8, dp-attn: true } dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ff6143d22d..4ef80fcf62 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5987,6 +5987,5 @@ scenario-type: - agentic-coding description: - - "Add GB200 MiniMax-M3 NVFP4 AgentX with EAGLE3-GQA, the B200 TP4 baseline, DEP4/DEP8, KV-routed P/D, and high-concurrency SimpleCPU offload." - - "Use vLLM v0.27.1, NVIDIA/srt-slurm v1.0.50, and Dynamo v1.3.1 with full 1M context, MiniMax parsing, KV-cache token metrics, and every logical vLLM metrics endpoint." + - "Add the tuned GB200 MiniMax-M3 FP4 AgentX frontier with EAGLE3, the B200 TP4 baseline, DEP4/DEP8, SimpleCPU offload, and KV-routed P/D." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609 From 1cf03b2b18737166e8aa26a07d00b9932f64dfe0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 15 Aug 2026 00:28:58 -0500 Subject: [PATCH 13/14] Stabilize GB200 MiniMax TP4 offload graphs --- .../gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml | 1 + perf-changelog.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml index 247690d2ab..1d62599edd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-vllm-simple-agentic.yaml @@ -58,6 +58,7 @@ backend: speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' stream-interval: 20 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' max-cudagraph-capture-size: 512 max-num-batched-tokens: 16384 no-enable-flashinfer-autotune: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4ef80fcf62..8b77e0a719 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5988,4 +5988,5 @@ - agentic-coding description: - "Add the tuned GB200 MiniMax-M3 FP4 AgentX frontier with EAGLE3, the B200 TP4 baseline, DEP4/DEP8, SimpleCPU offload, and KV-routed P/D." + - "Use full-decode-only CUDA graphs for TP4 SimpleCPU stability." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609 From 42d221d208f0968aac76967b77bcb7f7b17ecf3a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 15 Aug 2026 02:08:02 -0500 Subject: [PATCH 14/14] Stabilize GB200 MiniMax resident TP4 graphs --- .../vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml | 1 + perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml index aea6207c14..b74dd665b4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/gb200-fp4/agentic/agg-tp4-agentic.yaml @@ -63,6 +63,7 @@ backend: enable-prefix-caching: true kv-cache-metrics: true attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8","minimax_m3_msa_decode_backend":"cutlass"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' block-size: 128 gpu-memory-utilization: 0.9 max-model-len: 1048576 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8b77e0a719..436b627f85 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5988,5 +5988,5 @@ - agentic-coding description: - "Add the tuned GB200 MiniMax-M3 FP4 AgentX frontier with EAGLE3, the B200 TP4 baseline, DEP4/DEP8, SimpleCPU offload, and KV-routed P/D." - - "Use full-decode-only CUDA graphs for TP4 SimpleCPU stability." + - "Use full-decode-only CUDA graphs for TP4 stability." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2609