From f94cabe247ec0cf044d60534319dfaab2d2e9c54 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 3 Aug 2026 11:27:57 -0500 Subject: [PATCH 01/32] perf(agentx): add B200 Kimi K3 DSpark refresh --- .../agg-b200-tp8pp2-dspark-agentic.yaml | 143 ++++++++++++++++++ configs/nvidia-master.yaml | 32 ++++ perf-changelog.yaml | 5 + 3 files changed, 180 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml new file mode 100644 index 0000000000..49f1f0b0dd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml @@ -0,0 +1,143 @@ +name: "kimik3-vllm-agg-b200-tp8pp2-dspark-agentic" + +# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). +# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not +# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the +# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is +# deliberately off, so the 896 routed experts are TP-sharded inside each +# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. +# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in +# the kimi-k3 bring-up image. +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the +# multinode extension on functionstackx/srt-slurm-nv branch +# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so +# no Dynamo frontend/worker is involved and no dynamo install is needed. +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns + # the public port; rank-1 runs a headless engine worker (vLLM-native + # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the + # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_SERVER_DEV_MODE: "1" + # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window + # generous, and let one long AgentX request hold a PP stage beyond vLLM's + # 300-second model-execution default. + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + # Prefix-cache retention (variant G): 0, on the otherwise-unchanged + # GPU-resident variant D config. Any positive value must be a multiple of + # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' + # 32768 is hard-rejected at engine init — verified on this PR family), so + # 0 is the only value below one 3.1M-token scheduler block. + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value + # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a + # scheduler_block_size of 3145728 and the interval must be a multiple of + # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative + # and a multiple of scheduler_block_size (3145728)"). Default retention + # served fine in earlier runs. + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a + # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first + # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first + # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 + # matches the GB200/GB300 agentic recipes. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — + # legitimate here because this recipe serves directly with vllm serve + # (frontend.type: vllm), not through the dynamo worker entrypoint that + # rejects them. + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + # DSpark level 7 with probabilistic drafting and the committed golden + # synthetic acceptance length measured for Kimi-K3 at this draft length. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + # No explicit max-model-len: let vLLM derive the native 1M window from + # the model config (agentic trajectories blow past any small cap, and + # K3's KDA layers keep per-token KV small — only the 24 gated-MLA + # layers hold cache). Prefix caching stays on (default) for trajectory + # reuse. Cap prefill chunks so a single long request cannot OOM a + # pipeline stage; let vLLM pick max-num-seqs. + max-num-batched-tokens: 8192 + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep the aggregate worker in the multinode result schema so ingestion + # uses the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + # aiperf's conv-aware routing emits nvext.session_control, a removed POC + # field this dynamo build 400-rejects at warmup (schema moved to + # router/routing_constraints/agent_hints). Same opt-out as the GB300 + # aggregate AgentX recipes — and with a single aggregate worker there is + # no P/D routing to bind anyway. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 90e54195b6..4fdafcea1f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9253,6 +9253,38 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false +# Kimi-K3 MXFP4 B200 DSpark variant of the aggregated TP8 x PP2 topology. +# The server recipe applies the DSpark draft configuration; `mtp` selects the +# repository's speculative-decoding benchmark path and result identity. +kimik3-fp4-b200-dynamo-vllm-agentic-dspark: + image: vllm/vllm-openai:kimi-k3 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b200-dgxc + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [1, 2, 4, 8, 16, 32] + prefill: + num-worker: 1 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false + qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7a064f8090..2b7d6e3703 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5374,3 +5374,8 @@ - "Image: lmsysorg/sglang:v0.5.16-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420 +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + description: + - "Initial B200 Kimi K3 DSpark submission with corrected AgentX harness" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/0 From 256df24ace82b7c009948369d4896c942902e795 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 3 Aug 2026 11:28:22 -0500 Subject: [PATCH 02/32] chore: link B200 DSpark refresh PR --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2b7d6e3703..9383252290 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5378,4 +5378,4 @@ - kimik3-fp4-b200-dynamo-vllm-agentic-dspark description: - "Initial B200 Kimi K3 DSpark submission with corrected AgentX harness" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/0 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 From 9515848a301da0173ce60dbeea7243af1df246c4 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:31:55 -0500 Subject: [PATCH 03/32] feat: add b200 kimi k3 agentx profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：添加 B200 Kimi K3 AgentX 延迟、均衡吞吐、GPU 常驻高并发和 CPU KV 卸载配置，并通过 srt-slurm 启动多节点聚合式推理。 --- .../configs/kimik3-dspark-config-compat.sh | 47 ++++++ ...-b200-dep16-throughput-dspark-agentic.yaml | 124 +++++++++++++++ ...ut-vllm-simple-offload-dspark-agentic.yaml | 125 +++++++++++++++ ...gg-b200-tep16-balanced-dspark-agentic.yaml | 103 +++++++++++++ .../agg-b200-tp16-latency-dspark-agentic.yaml | 105 +++++++++++++ .../agg-b200-tp8pp2-dspark-agentic.yaml | 143 ------------------ configs/nvidia-master.yaml | 70 +++++++-- perf-changelog.yaml | 4 +- runners/launch_b200-dgxc.sh | 22 ++- 9 files changed, 577 insertions(+), 166 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh new file mode 100755 index 0000000000..1822eadc9a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Dynamo serializes the draft config without Kimi K3's config-class alias. +# Prepare a local metadata view that exposes the equivalent parallel-draft token +# without changing the downloaded checkpoint or vLLM source. +python3 - <<'PY' +import json +import os +from pathlib import Path + +from huggingface_hub import snapshot_download + +repo_id = "Inferact/Kimi-K3-DSpark" +target = Path("/tmp/Kimi-K3-DSpark") +snapshot = Path(snapshot_download(repo_id=repo_id)) +target.mkdir(parents=True, exist_ok=True) + +for source in snapshot.iterdir(): + if source.name == "config.json": + continue + destination = target / source.name + if destination.is_symlink(): + if destination.resolve() == source.resolve(): + continue + destination.unlink() + elif destination.exists(): + raise RuntimeError(f"Refusing to replace non-symlink path: {destination}") + destination.symlink_to(source) + +config = json.loads((snapshot / "config.json").read_text()) +mask_token_id = config.get("mask_token_id") +if not isinstance(mask_token_id, int): + raise RuntimeError(f"{repo_id} config is missing integer mask_token_id") + +pard_token = config.get("pard_token") +if pard_token not in (None, mask_token_id): + raise RuntimeError( + f"{repo_id} pard_token={pard_token} disagrees with mask_token_id={mask_token_id}" + ) +config["pard_token"] = mask_token_id + +temporary = target / "config.json.tmp" +temporary.write_text(json.dumps(config, indent=2) + "\n") +os.replace(temporary, target / "config.json") +print(f"Prepared {repo_id} compatibility view at {target}") +PY diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml new file mode 100644 index 0000000000..e14c238acf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml @@ -0,0 +1,124 @@ +name: "kimik3-vllm-agg-b200-dep16-throughput-dspark-agentic" + +# Official throughput-oriented multi_node_dep profile for two 8xB200 nodes. +# One Dynamo vLLM process per node owns eight local DP ranks; srt-slurm derives +# start ranks 0 and 8 from the allocated topology and EP spans all 16 GPUs. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "dynamo" + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 2 + kv-cache-block-size: 64 + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + data-parallel-size: 16 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: deep_gemm_mega_moe + all2all-backend: deepep_v2 + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + dyn-tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 + language-model-only: true + block-size: 64 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AGENTIC_WARMUP_GRACE_PERIOD: "3600" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml new file mode 100644 index 0000000000..c1d8a6a651 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml @@ -0,0 +1,125 @@ +name: "kimik3-vllm-agg-b200-dep16-simple-offload-dspark-agentic" + +# Official multi_node_dep profile with SimpleCPUOffloadConnector. Each of the +# 16 DP ranks reserves the official 220 GiB host-KV pool. Eight ranks consume +# 1.72 TiB on each B200 node, below the runner's configured host-DRAM budget. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "dynamo" + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 2 + kv-cache-block-size: 64 + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + data-parallel-size: 16 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: deep_gemm_mega_moe + all2all-backend: deepep_v2 + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + dyn-tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 + language-model-only: true + block-size: 64 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AGENTIC_WARMUP_GRACE_PERIOD: "3600" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml new file mode 100644 index 0000000000..6641c4d1b0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml @@ -0,0 +1,103 @@ +name: "kimik3-vllm-agg-b200-tep16-balanced-dspark-agentic" + +# Official balanced multi_node_tep profile for two 8xB200 nodes. TP16 shards +# attention and dense layers; EP16 shards the routed experts. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tep +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + all2all-backend: deepep_v2 + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 32 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml new file mode 100644 index 0000000000..6f7f2e8580 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -0,0 +1,105 @@ +name: "kimik3-vllm-agg-b200-tp16-latency-dspark-agentic" + +# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. +# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can + # OOM at the official 0.95 setting. Preserve the observed-safe headroom. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Performance runs use the committed K7 golden synthetic AL. The official + # recipe's real-verification form uses rejection_sample_method=block. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml deleted file mode 100644 index 49f1f0b0dd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-dspark-agentic" - -# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). -# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not -# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the -# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is -# deliberately off, so the 896 routed experts are TP-sharded inside each -# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. -# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. -# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in -# the kimi-k3 bring-up image. -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - -# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the -# multinode extension on functionstackx/srt-slurm-nv branch -# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so -# no Dynamo frontend/worker is involved and no dynamo install is needed. -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 1440 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns - # the public port; rank-1 runs a headless engine worker (vLLM-native - # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the - # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_SERVER_DEV_MODE: "1" - # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window - # generous, and let one long AgentX request hold a PP stage beyond vLLM's - # 300-second model-execution default. - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - # Prefix-cache retention (variant G): 0, on the otherwise-unchanged - # GPU-resident variant D config. Any positive value must be a multiple of - # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' - # 32768 is hard-rejected at engine init — verified on this PR family), so - # 0 is the only value below one 3.1M-token scheduler block. - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value - # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a - # scheduler_block_size of 3145728 and the interval must be a multiple of - # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative - # and a multiple of scheduler_block_size (3145728)"). Default retention - # served fine in earlier runs. - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a - # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first - # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first - # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 - # matches the GB200/GB300 agentic recipes. - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — - # legitimate here because this recipe serves directly with vllm serve - # (frontend.type: vllm), not through the dynamo worker entrypoint that - # rejects them. - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - # DSpark level 7 with probabilistic drafting and the committed golden - # synthetic acceptance length measured for Kimi-K3 at this draft length. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - # No explicit max-model-len: let vLLM derive the native 1M window from - # the model config (agentic trajectories blow past any small cap, and - # K3's KDA layers keep per-token KV small — only the 24 gated-MLA - # layers hold cache). Prefix caching stays on (default) for trajectory - # reuse. Cap prefill chunks so a single long request cannot OOM a - # pipeline stage; let vLLM pick max-num-seqs. - max-num-batched-tokens: 8192 - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep the aggregate worker in the multinode result schema so ingestion - # uses the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - # aiperf's conv-aware routing emits nvext.session_control, a removed POC - # field this dynamo build 400-rejects at warmup (schema moved to - # router/routing_constraints/agent_hints). Same opt-out as the GB300 - # aggregate AgentX recipes — and with a single aggregate worker there is - # no P/D routing to bind anyway. - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 4fdafcea1f..fa846bde98 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9253,9 +9253,12 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false -# Kimi-K3 MXFP4 B200 DSpark variant of the aggregated TP8 x PP2 topology. -# The server recipe applies the DSpark draft configuration; `mtp` selects the -# repository's speculative-decoding benchmark path and result identity. +# Kimi K3 B200 AgentX DSpark profiles derived from the official vLLM B200 +# strategies. TP16 and TEP16 use direct multi-node vLLM through srt-slurm; +# DEP16 uses srt-slurm's Dynamo aggregate path so node-local DP start ranks are +# derived from the allocation. All four recipes use the committed K7 synthetic +# acceptance length 3.84 for comparable performance runs. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200 kimik3-fp4-b200-dynamo-vllm-agentic-dspark: image: vllm/vllm-openai:kimi-k3 model: moonshotai/Kimi-K3 @@ -9267,23 +9270,70 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: disagg: false scenarios: agentic-coding: - - search-space: + - dram-utilization: 0.61 + search-space: + # Low-latency and high-interactivity profile. - spec-decoding: mtp - conc-list: [1, 2, 4, 8, 16, 32] + conc-list: [1, 2, 4, 8] prefill: num-worker: 1 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" decode: num-worker: 0 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false + # Balanced medium-concurrency TEP16 profile. + - spec-decoding: mtp + conc-list: [8, 16, 24, 32] + prefill: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 16 + ep: 16 + dp-attn: false + # High-concurrency GPU-resident DEP16 profile. + - spec-decoding: mtp + conc-list: [32, 64, 96, 128, 192, 256] + prefill: + num-worker: 1 + tp: 1 + ep: 16 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 1 + ep: 16 + dp-attn: true + # Maximum-concurrency DEP16 profile with 220 GiB of CPU KV per rank. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple, version: "13c59a3" } + conc-list: [128, 192, 256, 384] + prefill: + num-worker: 1 + tp: 1 + ep: 16 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 1 + ep: 16 + dp-attn: true qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9383252290..cd348e0b9e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5377,5 +5377,7 @@ - config-keys: - kimik3-fp4-b200-dynamo-vllm-agentic-dspark description: - - "Initial B200 Kimi K3 DSpark submission with corrected AgentX harness" + - "Add official Kimi K3 B200 AgentX profiles for TP16 latency, TEP16 balanced throughput, and DEP16 high concurrency through checked-in srt-slurm recipes" + - "Add a DEP16 SimpleCPUOffloadConnector profile with the official 220 GiB host-KV pool per rank" + - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 16d3338d09..bf759a010a 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -107,21 +107,19 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -rf "$SRT_REPO_DIR" fi - # TODO(CJQ): make first class upon srt-slurm upstream refactor + # Kimi K3 aggregate profiles use the srt-slurm fork that supports direct + # multi-node vLLM for TP/TEP and topology-derived per-node DP ranks for + # Dynamo DEP. Pin the tested renderer so branch movement cannot change + # generated rank commands between sweep points. if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then - # Direct-vLLM agentic experiment (Variant D): srt-slurm PR #278 - # (kylliang/direct-aggregate-vllm) adds frontend.type: vllm — `vllm - # serve` owns the OpenAI port itself, no Dynamo layer. The fork branch - # carries PR #278 plus the multi-node extension (vLLM-native - # --master-addr/--nnodes/--node-rank serve + headless non-leader - # ranks) so the 2-node TP8xPP2 topology can run. git clone --branch klaud/direct-vllm-multinode --single-branch https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 - if [[ $MODEL_PREFIX == "kimik3" ]]; then - mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic || exit 1 - fi + git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 || exit 1 + mkdir -p recipes/vllm/kimi-k3/agentic configs || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic || exit 1 + cp "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh" \ + configs/kimik3-dspark-config-compat.sh || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 From c5fa8dac0ef9785fc1c07968e2bb330251da7414 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:19:33 -0500 Subject: [PATCH 04/32] fix: bypass unsupported b200 custom collectives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：绕过 B200 上不受支持的自定义集合通信，避免 TP16 在权重加载前停滞。 --- .../kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 6f7f2e8580..e0ffba22cc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -69,6 +69,9 @@ backend: # OOM at the official 0.95 setting. Preserve the observed-safe headroom. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true + # This B200 pool has no cross-node MNNVL multicast. Explicitly bypass + # custom collective initialization, which otherwise stalls before load. + disable-custom-all-reduce: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' From 88d3bb7262403137a5915e3bc11403f47accdc29 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:34:39 -0500 Subject: [PATCH 05/32] fix: correct b200 dep launch accounting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：修正 B200 DEP 启动参数、KV 事件发布、GPU 计数与 CPU DRAM 预算，并禁用跨节点不受支持的 FlashInfer 集合通信。 --- ...-b200-dep16-throughput-dspark-agentic.yaml | 3 +- ...ut-vllm-simple-offload-dspark-agentic.yaml | 3 +- ...gg-b200-tep16-balanced-dspark-agentic.yaml | 3 +- .../agg-b200-tp16-latency-dspark-agentic.yaml | 6 +-- .../aggregation/process_agentic_result.py | 37 ++++++++++++-- .../test_process_agentic_result.py | 29 +++++++++++ utils/matrix_logic/generate_sweep_configs.py | 39 ++++++++------- .../test_generate_sweep_configs.py | 50 +++++++++++++++++++ 8 files changed, 141 insertions(+), 29 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml index e14c238acf..a549b7d04e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml @@ -59,6 +59,8 @@ backend: type: vllm connector: null dp_launch_mode: per_node + kv_events_config: + aggregated: true aggregated_environment: VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" @@ -89,7 +91,6 @@ backend: kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' enable-prefix-caching: true - enable-auto-tool-choice: true dyn-tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 dyn-reasoning-parser: kimi_k3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml index c1d8a6a651..c1a0899688 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml @@ -59,6 +59,8 @@ backend: type: vllm connector: null dp_launch_mode: per_node + kv_events_config: + aggregated: true aggregated_environment: VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" @@ -89,7 +91,6 @@ backend: kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' enable-prefix-caching: true - enable-auto-tool-choice: true dyn-tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 dyn-reasoning-parser: kimi_k3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml index 6641c4d1b0..557ca3a9ce 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml @@ -44,7 +44,8 @@ backend: connector: null aggregated_environment: VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" + # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index e0ffba22cc..57f29a1d88 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -44,7 +44,8 @@ backend: connector: null aggregated_environment: VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" + # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" @@ -69,9 +70,6 @@ backend: # OOM at the official 0.95 setting. Preserve the observed-safe headroom. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true - # This B200 pool has no cross-node MNNVL multicast. Explicitly bypass - # custom collective initialization, which otherwise stalls before load. - disable-custom-all-reduce: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' diff --git a/utils/agentic/aggregation/process_agentic_result.py b/utils/agentic/aggregation/process_agentic_result.py index 5d13fa0f1b..4971a132ba 100644 --- a/utils/agentic/aggregation/process_agentic_result.py +++ b/utils/agentic/aggregation/process_agentic_result.py @@ -95,6 +95,18 @@ def _validate_kv_offload_env() -> tuple[str, dict[str, str] | None]: return kv_offloading, backend_metadata +def _parallel_gpu_count( + tp: int, + pp: int, + pcp_size: int, + ep: int, + dp_attention: bool, +) -> int: + """Return the physical GPU count for one worker replica.""" + tensor_gpu_count = tp * pp * pcp_size + return max(tensor_gpu_count, ep) if dp_attention else tensor_gpu_count + + def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: is_multinode = env_bool("IS_MULTINODE") tp = env_int("TP", 1) @@ -111,7 +123,10 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: "PP_SIZE, DCP_SIZE, and PCP_SIZE must be positive integers." ) fields.update({"pp": pp, "dcp_size": dcp_size, "pcp_size": pcp_size}) - return fields, tp * pp * pcp_size, tp, ep, dp_attention + num_gpus = _parallel_gpu_count( + tp, pp, pcp_size, ep, env_bool("DP_ATTENTION") + ) + return fields, num_gpus, tp, ep, dp_attention prefill_num_workers = env_int("PREFILL_NUM_WORKERS") prefill_tp = env_int("PREFILL_TP") @@ -120,6 +135,7 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: prefill_pcp_size = env_int("PREFILL_PCP_SIZE", 1) prefill_ep = env_int("PREFILL_EP", 1) prefill_dp_attention = os.environ.get("PREFILL_DP_ATTN", "false") + prefill_dp_attention_enabled = env_bool("PREFILL_DP_ATTN") decode_num_workers = env_int("DECODE_NUM_WORKERS") decode_tp = env_int("DECODE_TP") decode_pp = env_int("DECODE_PP_SIZE", 1) @@ -127,6 +143,7 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: decode_pcp_size = env_int("DECODE_PCP_SIZE", 1) decode_ep = env_int("DECODE_EP", 1) decode_dp_attention = os.environ.get("DECODE_DP_ATTN", "false") + decode_dp_attention_enabled = env_bool("DECODE_DP_ATTN") worker_parallelism = ( prefill_pp, prefill_dcp_size, @@ -145,8 +162,20 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: raise SystemExit( "PREFILL_HARDWARE and DECODE_HARDWARE must be specified together." ) - num_prefill_gpu = prefill_num_workers * prefill_tp * prefill_pp * prefill_pcp_size - num_decode_gpu = decode_num_workers * decode_tp * decode_pp * decode_pcp_size + num_prefill_gpu = prefill_num_workers * _parallel_gpu_count( + prefill_tp, + prefill_pp, + prefill_pcp_size, + prefill_ep, + prefill_dp_attention_enabled, + ) + num_decode_gpu = decode_num_workers * _parallel_gpu_count( + decode_tp, + decode_pp, + decode_pcp_size, + decode_ep, + decode_dp_attention_enabled, + ) num_gpus = num_prefill_gpu + num_decode_gpu # Aggregated configs set decode num-worker 0 (prefill+decode co-located on one # worker), so there are no separate decode GPUs. Mirror process_result.py and drop @@ -162,7 +191,7 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: ep = max(prefill_ep, decode_ep) dp_attention = ( "true" - if env_bool("PREFILL_DP_ATTN") or env_bool("DECODE_DP_ATTN") + if prefill_dp_attention_enabled or decode_dp_attention_enabled else "false" ) fields.update( diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index 21a7307630..8bb4d36b06 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -564,6 +564,35 @@ def test_multinode_processor_surfaces_heterogeneous_hardware(tmp_path: Path): agg["num_decode_gpu"], ) == (2, 4, 1, 32) +def test_multinode_processor_counts_attention_dp_ranks( + monkeypatch: pytest.MonkeyPatch, +): + env = { + "IS_MULTINODE": "true", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "1", + "PREFILL_PP_SIZE": "1", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "16", + "PREFILL_DP_ATTN": "true", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "1", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "1", + "DECODE_DP_ATTN": "false", + } + for name, value in env.items(): + monkeypatch.setenv(name, value) + + fields, num_gpus, tp, ep, dp_attention = _gpu_shape() + + assert fields["num_prefill_gpu"] == 16 + assert fields["num_decode_gpu"] == 0 + assert (num_gpus, tp, ep, dp_attention) == (16, 1, 16, "true") + def test_multinode_processor_omits_homogeneous_hardware(tmp_path: Path): result_dir = _write_fixture(tmp_path) diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 86f192b1c8..6ccd84b604 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -82,13 +82,21 @@ def runner_gpus_per_node(runner: str, runner_data: dict) -> int: return runner_hardware_int(runner, runner_data, Fields.GPUS_PER_NODE.value) -def effective_gpu_count(benchmark: dict) -> int: - """Return GPUs used by a single-node TP/PP/PCP topology.""" - return ( - benchmark[Fields.TP.value] - * benchmark.get(Fields.PP.value, 1) - * benchmark.get(Fields.PCP_SIZE.value, 1) +def _parallel_gpu_count(config: dict) -> int: + """Return the GPU footprint encoded by one parallelism configuration.""" + tensor_gpu_count = ( + config[Fields.TP.value] + * config.get(Fields.PP.value, 1) + * config.get(Fields.PCP_SIZE.value, 1) ) + if config.get(Fields.DP_ATTN.value, False): + return max(tensor_gpu_count, config.get(Fields.EP.value, 1)) + return tensor_gpu_count + + +def effective_gpu_count(benchmark: dict) -> int: + """Return GPUs used by a single-node parallel topology.""" + return _parallel_gpu_count(benchmark) def with_worker_parallelism_defaults(worker: dict) -> dict: """Return a worker config with explicit parallelism defaults.""" @@ -109,29 +117,24 @@ def worker_gpus_per_node(worker: dict, gpus_per_node: int) -> int: are rejected rather than silently truncated, keeping parity with the single-node "must fit the node" rule: - * A replica larger than one node (tp*pp*pcp > gpus-per-node) must fill whole - nodes, i.e. be an exact multiple of gpus-per-node; each of its nodes is - then fully occupied (fraction 1). + * A replica larger than one node must fill whole nodes, i.e. be an exact + multiple of gpus-per-node; each of its nodes is then fully occupied. * A replica within one node must divide it evenly so co-located replicas of the same role tile the node without overlap. + * Attention-DP topologies include the EP-sized independent rank pool. """ - gpus_per_replica = ( - worker[Fields.TP.value] - * worker.get(Fields.PP.value, 1) - * worker.get(Fields.PCP_SIZE.value, 1) - ) + gpus_per_replica = _parallel_gpu_count(worker) if gpus_per_replica > gpus_per_node: if gpus_per_replica % gpus_per_node != 0: raise ValueError( - f"worker {Fields.TP.value}*{Fields.PP.value}*{Fields.PCP_SIZE.value}" - f"={gpus_per_replica} spans multiple nodes but is not a multiple " + f"worker GPU footprint={gpus_per_replica} spans multiple nodes " + "but is not a multiple " f"of {Fields.GPUS_PER_NODE.value}={gpus_per_node}" ) return gpus_per_node if gpus_per_node % gpus_per_replica != 0: raise ValueError( - f"worker {Fields.TP.value}*{Fields.PP.value}*{Fields.PCP_SIZE.value}" - f"={gpus_per_replica} does not divide " + f"worker GPU footprint={gpus_per_replica} does not divide " f"{Fields.GPUS_PER_NODE.value}={gpus_per_node} evenly" ) return gpus_per_replica diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 7cbfea79af..47974eb268 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2274,6 +2274,56 @@ def test_multinode_agentic_preserves_kv_offload_fields(self, sample_runner_confi # DRAM: 2861022 MiB * 0.80. assert result[0]["total-cpu-dram-gb"] == 2399 + def test_multinode_agentic_dep_budget_counts_attention_dp_ranks( + self, sample_runner_config + ): + config = { + "kimik3-dep": { + "image": "vllm/vllm-openai:kimi-k3", + "model": "moonshotai/Kimi-K3", + "model-prefix": "kimik3", + "precision": "fp4", + "framework": "dynamo-vllm", + "runner": "cluster:b200-dgxc", + "multinode": True, + "scenarios": { + "agentic-coding": [{ + "dram-utilization": 0.61, + "search-space": [{ + "conc-list": [128], + "kv-offloading": "dram", + "kv-offload-backend": {"name": "vllm-simple"}, + "prefill": { + "num-worker": 1, + "tp": 1, + "ep": 16, + "dp-attn": True, + }, + "decode": { + "num-worker": 0, + "tp": 1, + "ep": 1, + "dp-attn": False, + }, + }], + }], + }, + }, + } + args = argparse.Namespace( + config_keys=["kimik3-dep"], + seq_lens=None, + conc=None, + scenario_type=["agentic-coding"], + runner_node_filter=None, + ) + + result = generate_test_config_sweep(args, config, sample_runner_config) + + assert len(result) == 1 + # DEP16 spans both nodes, so all eight ranks on each node share DRAM. + assert result[0]["total-cpu-dram-gb"] == 1829 + def test_multinode_agentic_budget_ignores_decode_topology( self, sample_runner_config ): From 3cb4971232d0b16bde3d3b511b652c4bc97249cd Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:54:01 -0500 Subject: [PATCH 06/32] fix: force pynccl for cross-node tp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：为跨节点 TP16 和 TEP16 强制使用 PyNCCL，跳过会在 B200 集群上停滞的 MNNVL 自定义集合通信初始化。 --- .../agentic/agg-b200-tep16-balanced-dspark-agentic.yaml | 3 +++ .../kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml index 557ca3a9ce..ec7516160a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml @@ -70,6 +70,9 @@ backend: all2all-backend: deepep_v2 gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true + # The generic custom communicator also probes MNNVL and stalls before + # model load on this pool. Use PyNCCL for the cross-node TP group. + disable-custom-all-reduce: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 57f29a1d88..81890ce47b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -70,6 +70,9 @@ backend: # OOM at the official 0.95 setting. Preserve the observed-safe headroom. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true + # The generic custom communicator also probes MNNVL and stalls before + # model load on this pool. Use PyNCCL for the cross-node TP group. + disable-custom-all-reduce: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' From 94d677aeb370a4b5033878cb1357f1849ff86ab6 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:03:02 -0500 Subject: [PATCH 07/32] fix: disable cross-node latent tail fusion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：为跨节点 TP16 和 TEP16 禁用 Kimi K3 LatentMoE tail fusion，避免 torch symmetric memory 无法跨节点传递文件描述符。 --- .../agentic/agg-b200-tep16-balanced-dspark-agentic.yaml | 4 +++- .../kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml index ec7516160a..15ef2a7386 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml @@ -43,7 +43,9 @@ backend: type: vllm connector: null aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + # K3's fused tail uses torch symmetric memory, which cannot exchange file + # descriptors across this two-node TP group. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 81890ce47b..42a9b25371 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -43,7 +43,9 @@ backend: type: vllm connector: null aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + # K3's fused tail uses torch symmetric memory, which cannot exchange file + # descriptors across this two-node TP group. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" From 349ddfdfa0e3f307cb2eb3503fadfde24b52b479 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:45:10 -0500 Subject: [PATCH 08/32] fix: disable cross-node allreduce fusion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：为跨节点 TP16 和 TEP16 禁用 allreduce/RMS fusion，避免 profile_run 再次选择不受支持的 FlashInfer MNNVL 工作区。 --- .../agentic/agg-b200-tep16-balanced-dspark-agentic.yaml | 3 +++ .../kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml index 15ef2a7386..b7b0616811 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml @@ -75,6 +75,9 @@ backend: # The generic custom communicator also probes MNNVL and stalls before # model load on this pool. Use PyNCCL for the cross-node TP group. disable-custom-all-reduce: true + # The default allreduce/RMS fusion independently allocates a FlashInfer + # MNNVL workspace during profile_run, bypassing the communicator flags. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 42a9b25371..7163e44a8e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -75,6 +75,9 @@ backend: # The generic custom communicator also probes MNNVL and stalls before # model load on this pool. Use PyNCCL for the cross-node TP group. disable-custom-all-reduce: true + # The default allreduce/RMS fusion independently allocates a FlashInfer + # MNNVL workspace during profile_run, bypassing the communicator flags. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' From 48dffa17d08d9279877a4af1d30ad3b408c04a21 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:43:23 -0500 Subject: [PATCH 09/32] fix: use viable b200 tp8pp2 profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：改用可运行的 B200 TP8 x PP2 配置，并移除无法启动的跨节点 TP16、TEP16 和 DEP16 配置。 --- ...-b200-dep16-throughput-dspark-agentic.yaml | 125 --------------- ...ut-vllm-simple-offload-dspark-agentic.yaml | 126 --------------- ...gg-b200-tep16-balanced-dspark-agentic.yaml | 112 ------------- .../agg-b200-tp16-latency-dspark-agentic.yaml | 114 -------------- .../agg-b200-tp8pp2-dspark-agentic.yaml | 143 +++++++++++++++++ ...p2-vllm-simple-offload-dspark-agentic.yaml | 147 ++++++++++++++++++ configs/nvidia-master.yaml | 79 +++++----- perf-changelog.yaml | 4 +- 8 files changed, 335 insertions(+), 515 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml deleted file mode 100644 index a549b7d04e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: "kimik3-vllm-agg-b200-dep16-throughput-dspark-agentic" - -# Official throughput-oriented multi_node_dep profile for two 8xB200 nodes. -# One Dynamo vLLM process per node owns eight local DP ranks; srt-slurm derives -# start ranks 0 and 8 from the allocated topology and EP spans all 16 GPUs. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "dynamo" - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 2 - kv-cache-block-size: 64 - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: - aggregated: true - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - data-parallel-size: 16 - enable-expert-parallel: true - trust-remote-code: true - load-format: fastsafetensors - moe-backend: deep_gemm_mega_moe - all2all-backend: deepep_v2 - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' - enable-prefix-caching: true - dyn-tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 - language-model-only: true - block-size: 64 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml deleted file mode 100644 index c1a0899688..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml +++ /dev/null @@ -1,126 +0,0 @@ -name: "kimik3-vllm-agg-b200-dep16-simple-offload-dspark-agentic" - -# Official multi_node_dep profile with SimpleCPUOffloadConnector. Each of the -# 16 DP ranks reserves the official 220 GiB host-KV pool. Eight ranks consume -# 1.72 TiB on each B200 node, below the runner's configured host-DRAM budget. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "dynamo" - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 2 - kv-cache-block-size: 64 - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: - aggregated: true - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - data-parallel-size: 16 - enable-expert-parallel: true - trust-remote-code: true - load-format: fastsafetensors - moe-backend: deep_gemm_mega_moe - all2all-backend: deepep_v2 - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' - enable-prefix-caching: true - dyn-tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 - language-model-only: true - block-size: 64 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml deleted file mode 100644 index b7b0616811..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml +++ /dev/null @@ -1,112 +0,0 @@ -name: "kimik3-vllm-agg-b200-tep16-balanced-dspark-agentic" - -# Official balanced multi_node_tep profile for two 8xB200 nodes. TP16 shards -# attention and dense layers; EP16 shards the routed experts. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tep -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - # K3's fused tail uses torch symmetric memory, which cannot exchange file - # descriptors across this two-node TP group. - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" - # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. - VLLM_ALLREDUCE_USE_FLASHINFER: "0" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - enable-expert-parallel: true - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - all2all-backend: deepep_v2 - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - # The generic custom communicator also probes MNNVL and stalls before - # model load on this pool. Use PyNCCL for the cross-node TP group. - disable-custom-all-reduce: true - # The default allreduce/RMS fusion independently allocates a FlashInfer - # MNNVL workspace during profile_run, bypassing the communicator flags. - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' - max-model-len: 1048576 - kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' - enable-prefix-caching: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 32 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml deleted file mode 100644 index 7163e44a8e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp16-latency-dspark-agentic" - -# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. -# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - # K3's fused tail uses torch symmetric memory, which cannot exchange file - # descriptors across this two-node TP group. - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" - # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. - VLLM_ALLREDUCE_USE_FLASHINFER: "0" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can - # OOM at the official 0.95 setting. Preserve the observed-safe headroom. - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - # The generic custom communicator also probes MNNVL and stalls before - # model load on this pool. Use PyNCCL for the cross-node TP group. - disable-custom-all-reduce: true - # The default allreduce/RMS fusion independently allocates a FlashInfer - # MNNVL workspace during profile_run, bypassing the communicator flags. - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' - max-model-len: 1048576 - kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' - enable-prefix-caching: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 8 - max-num-batched-tokens: 8192 - # Performance runs use the committed K7 golden synthetic AL. The official - # recipe's real-verification form uses rejection_sample_method=block. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml new file mode 100644 index 0000000000..49f1f0b0dd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml @@ -0,0 +1,143 @@ +name: "kimik3-vllm-agg-b200-tp8pp2-dspark-agentic" + +# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). +# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not +# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the +# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is +# deliberately off, so the 896 routed experts are TP-sharded inside each +# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. +# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in +# the kimi-k3 bring-up image. +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the +# multinode extension on functionstackx/srt-slurm-nv branch +# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so +# no Dynamo frontend/worker is involved and no dynamo install is needed. +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns + # the public port; rank-1 runs a headless engine worker (vLLM-native + # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the + # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_SERVER_DEV_MODE: "1" + # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window + # generous, and let one long AgentX request hold a PP stage beyond vLLM's + # 300-second model-execution default. + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + # Prefix-cache retention (variant G): 0, on the otherwise-unchanged + # GPU-resident variant D config. Any positive value must be a multiple of + # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' + # 32768 is hard-rejected at engine init — verified on this PR family), so + # 0 is the only value below one 3.1M-token scheduler block. + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value + # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a + # scheduler_block_size of 3145728 and the interval must be a multiple of + # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative + # and a multiple of scheduler_block_size (3145728)"). Default retention + # served fine in earlier runs. + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a + # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first + # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first + # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 + # matches the GB200/GB300 agentic recipes. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — + # legitimate here because this recipe serves directly with vllm serve + # (frontend.type: vllm), not through the dynamo worker entrypoint that + # rejects them. + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + # DSpark level 7 with probabilistic drafting and the committed golden + # synthetic acceptance length measured for Kimi-K3 at this draft length. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + # No explicit max-model-len: let vLLM derive the native 1M window from + # the model config (agentic trajectories blow past any small cap, and + # K3's KDA layers keep per-token KV small — only the 24 gated-MLA + # layers hold cache). Prefix caching stays on (default) for trajectory + # reuse. Cap prefill chunks so a single long request cannot OOM a + # pipeline stage; let vLLM pick max-num-seqs. + max-num-batched-tokens: 8192 + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep the aggregate worker in the multinode result schema so ingestion + # uses the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + # aiperf's conv-aware routing emits nvext.session_control, a removed POC + # field this dynamo build 400-rejects at warmup (schema moved to + # router/routing_constraints/agent_hints). Same opt-out as the GB300 + # aggregate AgentX recipes — and with a single aggregate worker there is + # no P/D routing to bind anyway. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml new file mode 100644 index 0000000000..f179c13146 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml @@ -0,0 +1,147 @@ +name: "kimik3-vllm-agg-b200-tp8pp2-simple-offload-dspark-agentic" + +# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). +# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not +# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the +# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is +# deliberately off, so the 896 routed experts are TP-sharded inside each +# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. +# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in +# the kimi-k3 bring-up image. +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the +# multinode extension on functionstackx/srt-slurm-nv branch +# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so +# no Dynamo frontend/worker is involved and no dynamo install is needed. +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns + # the public port; rank-1 runs a headless engine worker (vLLM-native + # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the + # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_SERVER_DEV_MODE: "1" + # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window + # generous, and let one long AgentX request hold a PP stage beyond vLLM's + # 300-second model-execution default. + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + # Prefix-cache retention (variant G): 0, on the otherwise-unchanged + # GPU-resident variant D config. Any positive value must be a multiple of + # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' + # 32768 is hard-rejected at engine init — verified on this PR family), so + # 0 is the only value below one 3.1M-token scheduler block. + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value + # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a + # scheduler_block_size of 3145728 and the interval must be a multiple of + # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative + # and a multiple of scheduler_block_size (3145728)"). Default retention + # served fine in earlier runs. + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a + # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first + # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first + # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 + # matches the GB200/GB300 agentic recipes. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — + # legitimate here because this recipe serves directly with vllm serve + # (frontend.type: vllm), not through the dynamo worker entrypoint that + # rejects them. + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + # DSpark level 7 with probabilistic drafting and the committed golden + # synthetic acceptance length measured for Kimi-K3 at this draft length. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + # 220 GiB of host DRAM per TP rank extends the KV working set beyond + # the observed GPU-resident c16-c32 cliff. Eager offload avoids a first-hit + # allocation spike during long AgentX trajectories. + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + # No explicit max-model-len: let vLLM derive the native 1M window from + # the model config (agentic trajectories blow past any small cap, and + # K3's KDA layers keep per-token KV small — only the 24 gated-MLA + # layers hold cache). Prefix caching stays on (default) for trajectory + # reuse. Cap prefill chunks so a single long request cannot OOM a + # pipeline stage; let vLLM pick max-num-seqs. + max-num-batched-tokens: 8192 + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep the aggregate worker in the multinode result schema so ingestion + # uses the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + # aiperf's conv-aware routing emits nvext.session_control, a removed POC + # field this dynamo build 400-rejects at warmup (schema moved to + # router/routing_constraints/agent_hints). Same opt-out as the GB300 + # aggregate AgentX recipes — and with a single aggregate worker there is + # no P/D routing to bind anyway. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index fa846bde98..594b713e89 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9253,12 +9253,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false -# Kimi K3 B200 AgentX DSpark profiles derived from the official vLLM B200 -# strategies. TP16 and TEP16 use direct multi-node vLLM through srt-slurm; -# DEP16 uses srt-slurm's Dynamo aggregate path so node-local DP start ranks are -# derived from the allocation. All four recipes use the committed K7 synthetic -# acceptance length 3.84 for comparable performance runs. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200 +# Kimi K3's 2.8T MXFP4 weights require TP8 x PP2 across all 16 B200s. +# Cross-node TP16/TEP16 is unsupported on this pool because it lacks MNNVL +# multicast, while DEP16 replicates the full model and OOMs before weight load. +# Split the proven TP8 x PP2 topology into GPU-resident latency, balanced, and +# throughput tiers, plus a CPU KV-offloaded capacity tier. kimik3-fp4-b200-dynamo-vllm-agentic-dspark: image: vllm/vllm-openai:kimi-k3 model: moonshotai/Kimi-K3 @@ -9272,68 +9271,76 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: agentic-coding: - dram-utilization: 0.61 search-space: - # Low-latency and high-interactivity profile. + # Low-latency and high-interactivity points. - spec-decoding: mtp - conc-list: [1, 2, 4, 8] + conc-list: [1, 2, 4] prefill: num-worker: 1 - tp: 16 + tp: 8 + pp: 2 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" decode: num-worker: 0 - tp: 16 + tp: 8 + pp: 2 ep: 1 dp-attn: false - # Balanced medium-concurrency TEP16 profile. + # Balanced medium-concurrency points. - spec-decoding: mtp - conc-list: [8, 16, 24, 32] + conc-list: [8, 12, 16] prefill: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + pp: 2 + ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" decode: num-worker: 0 - tp: 16 - ep: 16 + tp: 8 + pp: 2 + ep: 1 dp-attn: false - # High-concurrency GPU-resident DEP16 profile. + # GPU-resident throughput points around the observed c16-c32 KV cliff. - spec-decoding: mtp - conc-list: [32, 64, 96, 128, 192, 256] + conc-list: [20, 24, 28, 32] prefill: num-worker: 1 - tp: 1 - ep: 16 - dp-attn: true + tp: 8 + pp: 2 + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" decode: num-worker: 0 - tp: 1 - ep: 16 - dp-attn: true - # Maximum-concurrency DEP16 profile with 220 GiB of CPU KV per rank. + tp: 8 + pp: 2 + ep: 1 + dp-attn: false + # CPU KV-offloaded capacity points beyond the GPU-resident cliff. - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple, version: "13c59a3" } - conc-list: [128, 192, 256, 384] + conc-list: [32, 48, 64] prefill: num-worker: 1 - tp: 1 - ep: 16 - dp-attn: true + tp: 8 + pp: 2 + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml" decode: num-worker: 0 - tp: 1 - ep: 16 - dp-attn: true + tp: 8 + pp: 2 + ep: 1 + dp-attn: false qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cd348e0b9e..ea815e0a30 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5377,7 +5377,7 @@ - config-keys: - kimik3-fp4-b200-dynamo-vllm-agentic-dspark description: - - "Add official Kimi K3 B200 AgentX profiles for TP16 latency, TEP16 balanced throughput, and DEP16 high concurrency through checked-in srt-slurm recipes" - - "Add a DEP16 SimpleCPUOffloadConnector profile with the official 220 GiB host-KV pool per rank" + - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the proven TP8 x PP2 topology" + - "Add a TP8 x PP2 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 From 4508808a9776006e137865b753a64f959bad6be2 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:59:56 -0500 Subject: [PATCH 10/32] fix: keep dspark on pp-free tp16 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：DSpark 草稿模型不支持流水线并行，因此改回无 PP 的 TP16 配置，并保留 GPU 与 CPU KV 容量档位。 --- .../agg-b200-tp16-latency-dspark-agentic.yaml | 114 ++++++++++++++ ...16-vllm-simple-offload-dspark-agentic.yaml | 115 ++++++++++++++ .../agg-b200-tp8pp2-dspark-agentic.yaml | 143 ----------------- ...p2-vllm-simple-offload-dspark-agentic.yaml | 147 ------------------ configs/nvidia-master.yaml | 44 +++--- perf-changelog.yaml | 4 +- 6 files changed, 249 insertions(+), 318 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml new file mode 100644 index 0000000000..7163e44a8e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -0,0 +1,114 @@ +name: "kimik3-vllm-agg-b200-tp16-latency-dspark-agentic" + +# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. +# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # K3's fused tail uses torch symmetric memory, which cannot exchange file + # descriptors across this two-node TP group. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" + # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can + # OOM at the official 0.95 setting. Preserve the observed-safe headroom. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # The generic custom communicator also probes MNNVL and stalls before + # model load on this pool. Use PyNCCL for the cross-node TP group. + disable-custom-all-reduce: true + # The default allreduce/RMS fusion independently allocates a FlashInfer + # MNNVL workspace during profile_run, bypassing the communicator flags. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Performance runs use the committed K7 golden synthetic AL. The official + # recipe's real-verification form uses rejection_sample_method=block. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml new file mode 100644 index 0000000000..391aef0a44 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml @@ -0,0 +1,115 @@ +name: "kimik3-vllm-agg-b200-tp16-simple-offload-dspark-agentic" + +# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. +# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # K3's fused tail uses torch symmetric memory, which cannot exchange file + # descriptors across this two-node TP group. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" + # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can + # OOM at the official 0.95 setting. Preserve the observed-safe headroom. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # The generic custom communicator also probes MNNVL and stalls before + # model load on this pool. Use PyNCCL for the cross-node TP group. + disable-custom-all-reduce: true + # The default allreduce/RMS fusion independently allocates a FlashInfer + # MNNVL workspace during profile_run, bypassing the communicator flags. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Performance runs use the committed K7 golden synthetic AL. The official + # recipe's real-verification form uses rejection_sample_method=block. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml deleted file mode 100644 index 49f1f0b0dd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-dspark-agentic" - -# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). -# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not -# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the -# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is -# deliberately off, so the 896 routed experts are TP-sharded inside each -# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. -# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. -# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in -# the kimi-k3 bring-up image. -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - -# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the -# multinode extension on functionstackx/srt-slurm-nv branch -# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so -# no Dynamo frontend/worker is involved and no dynamo install is needed. -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 1440 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns - # the public port; rank-1 runs a headless engine worker (vLLM-native - # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the - # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_SERVER_DEV_MODE: "1" - # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window - # generous, and let one long AgentX request hold a PP stage beyond vLLM's - # 300-second model-execution default. - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - # Prefix-cache retention (variant G): 0, on the otherwise-unchanged - # GPU-resident variant D config. Any positive value must be a multiple of - # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' - # 32768 is hard-rejected at engine init — verified on this PR family), so - # 0 is the only value below one 3.1M-token scheduler block. - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value - # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a - # scheduler_block_size of 3145728 and the interval must be a multiple of - # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative - # and a multiple of scheduler_block_size (3145728)"). Default retention - # served fine in earlier runs. - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a - # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first - # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first - # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 - # matches the GB200/GB300 agentic recipes. - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — - # legitimate here because this recipe serves directly with vllm serve - # (frontend.type: vllm), not through the dynamo worker entrypoint that - # rejects them. - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - # DSpark level 7 with probabilistic drafting and the committed golden - # synthetic acceptance length measured for Kimi-K3 at this draft length. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - # No explicit max-model-len: let vLLM derive the native 1M window from - # the model config (agentic trajectories blow past any small cap, and - # K3's KDA layers keep per-token KV small — only the 24 gated-MLA - # layers hold cache). Prefix caching stays on (default) for trajectory - # reuse. Cap prefill chunks so a single long request cannot OOM a - # pipeline stage; let vLLM pick max-num-seqs. - max-num-batched-tokens: 8192 - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep the aggregate worker in the multinode result schema so ingestion - # uses the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - # aiperf's conv-aware routing emits nvext.session_control, a removed POC - # field this dynamo build 400-rejects at warmup (schema moved to - # router/routing_constraints/agent_hints). Same opt-out as the GB300 - # aggregate AgentX recipes — and with a single aggregate worker there is - # no P/D routing to bind anyway. - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml deleted file mode 100644 index f179c13146..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml +++ /dev/null @@ -1,147 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-simple-offload-dspark-agentic" - -# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). -# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not -# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the -# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is -# deliberately off, so the 896 routed experts are TP-sharded inside each -# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. -# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. -# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in -# the kimi-k3 bring-up image. -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - -# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the -# multinode extension on functionstackx/srt-slurm-nv branch -# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so -# no Dynamo frontend/worker is involved and no dynamo install is needed. -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 1440 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns - # the public port; rank-1 runs a headless engine worker (vLLM-native - # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the - # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_SERVER_DEV_MODE: "1" - # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window - # generous, and let one long AgentX request hold a PP stage beyond vLLM's - # 300-second model-execution default. - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - # Prefix-cache retention (variant G): 0, on the otherwise-unchanged - # GPU-resident variant D config. Any positive value must be a multiple of - # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' - # 32768 is hard-rejected at engine init — verified on this PR family), so - # 0 is the only value below one 3.1M-token scheduler block. - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value - # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a - # scheduler_block_size of 3145728 and the interval must be a multiple of - # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative - # and a multiple of scheduler_block_size (3145728)"). Default retention - # served fine in earlier runs. - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a - # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first - # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first - # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 - # matches the GB200/GB300 agentic recipes. - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — - # legitimate here because this recipe serves directly with vllm serve - # (frontend.type: vllm), not through the dynamo worker entrypoint that - # rejects them. - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - # DSpark level 7 with probabilistic drafting and the committed golden - # synthetic acceptance length measured for Kimi-K3 at this draft length. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - # 220 GiB of host DRAM per TP rank extends the KV working set beyond - # the observed GPU-resident c16-c32 cliff. Eager offload avoids a first-hit - # allocation spike during long AgentX trajectories. - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' - # No explicit max-model-len: let vLLM derive the native 1M window from - # the model config (agentic trajectories blow past any small cap, and - # K3's KDA layers keep per-token KV small — only the 24 gated-MLA - # layers hold cache). Prefix caching stays on (default) for trajectory - # reuse. Cap prefill chunks so a single long request cannot OOM a - # pipeline stage; let vLLM pick max-num-seqs. - max-num-batched-tokens: 8192 - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep the aggregate worker in the multinode result schema so ingestion - # uses the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - # aiperf's conv-aware routing emits nvext.session_control, a removed POC - # field this dynamo build 400-rejects at warmup (schema moved to - # router/routing_constraints/agent_hints). Same opt-out as the GB300 - # aggregate AgentX recipes — and with a single aggregate worker there is - # no P/D routing to bind anyway. - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 594b713e89..f93560d511 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9253,11 +9253,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false -# Kimi K3's 2.8T MXFP4 weights require TP8 x PP2 across all 16 B200s. -# Cross-node TP16/TEP16 is unsupported on this pool because it lacks MNNVL -# multicast, while DEP16 replicates the full model and OOMs before weight load. -# Split the proven TP8 x PP2 topology into GPU-resident latency, balanced, and -# throughput tiers, plus a CPU KV-offloaded capacity tier. +# DSpark's K3DSparkModel does not implement pipeline parallelism, so Kimi K3 +# DSpark must use the official PP1 TP16 layout across all 16 B200s. Keep plain +# TP because the latency and capacity objectives do not require expert +# parallelism. Split the same serving topology into latency, balanced, +# throughput, and CPU KV-offloaded capacity tiers. kimik3-fp4-b200-dynamo-vllm-agentic-dspark: image: vllm/vllm-openai:kimi-k3 model: moonshotai/Kimi-K3 @@ -9276,16 +9276,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [1, 2, 4] prefill: num-worker: 1 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" decode: num-worker: 0 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false # Balanced medium-concurrency points. @@ -9293,33 +9291,29 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [8, 12, 16] prefill: num-worker: 1 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" decode: num-worker: 0 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false - # GPU-resident throughput points around the observed c16-c32 KV cliff. + # GPU-resident throughput points around the prior c16-c32 KV cliff. - spec-decoding: mtp conc-list: [20, 24, 28, 32] prefill: num-worker: 1 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" decode: num-worker: 0 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false # CPU KV-offloaded capacity points beyond the GPU-resident cliff. @@ -9329,16 +9323,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [32, 48, 64] prefill: num-worker: 1 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml" decode: num-worker: 0 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ea815e0a30..3f57ad75e2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5377,7 +5377,7 @@ - config-keys: - kimik3-fp4-b200-dynamo-vllm-agentic-dspark description: - - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the proven TP8 x PP2 topology" - - "Add a TP8 x PP2 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" + - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the PP-free TP16 DSpark topology" + - "Add a TP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 From 3bf761605db00ab09b0737cff7944d8ba771a392 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:15:43 -0500 Subject: [PATCH 11/32] fix: test B200 with modern Kimi nightly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：使用新版 Kimi nightly 镜像测试 B200 --- .../kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 4 ++-- .../agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 7163e44a8e..355231a6f7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -5,14 +5,14 @@ name: "kimik3-vllm-agg-b200-tp16-latency-dspark-agentic" # https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp model: path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" + container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:kimi-k3" + image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" dynamo: install: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml index 391aef0a44..c561167e49 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml @@ -5,14 +5,14 @@ name: "kimik3-vllm-agg-b200-tp16-simple-offload-dspark-agentic" # https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp model: path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" + container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" precision: "fp4" identity: model: repo: "moonshotai/Kimi-K3" container: - image: "vllm/vllm-openai:kimi-k3" + image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" dynamo: install: false diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b43ee988c2..6eb1f878e6 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7910,7 +7910,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic: # parallelism. Split the same serving topology into latency, balanced, # throughput, and CPU KV-offloaded capacity tiers. kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - image: vllm/vllm-openai:kimi-k3 + image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:b200-dgxc diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c0dfdb3006..a38cd30954 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5717,4 +5717,5 @@ - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the PP-free TP16 DSpark topology" - "Add a TP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" + - "Use the pinned 2026-08-10 vLLM nightly with upstream Kimi K3 and DSpark support" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 \ No newline at end of file From ef1ffac6876e044bbd614453fe3d75188dcaf0a6 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:31:33 -0500 Subject: [PATCH 12/32] fix: disable fused Kimi latent tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：禁用 Kimi 跨节点融合 latent-MoE 尾部路径，改用可移植的 PyNCCL 通信。 --- .../configs/kimik3-dspark-config-compat.sh | 45 +++++++++++++++++++ .../agg-b200-tp16-latency-dspark-agentic.yaml | 5 ++- ...16-vllm-simple-offload-dspark-agentic.yaml | 5 ++- perf-changelog.yaml | 1 + 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh index a42c6aba77..42ea3dadd6 100755 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh @@ -1,6 +1,51 @@ #!/usr/bin/env bash set -euo pipefail +# The pinned vLLM nightly unconditionally enables Kimi K3's SM100 latent-MoE +# tail fusion. That kernel uses torch symmetric memory, whose file-descriptor +# rendezvous is node-local and fails for TP16 across two B200 nodes. Patch the +# pinned implementation to honor the explicit portable-path opt-out below. +python3 - <<'PY' +import importlib.util +import os +from pathlib import Path + +flag = "INFERENCEX_DISABLE_K3_LATENT_MOE_TAIL_FUSION" +spec = importlib.util.find_spec("vllm") +if spec is None or spec.submodule_search_locations is None: + raise RuntimeError("Cannot locate the installed vLLM package") + +package = Path(next(iter(spec.submodule_search_locations))) +runner = package / "models/kimi_k3/nvidia/latent_moe_runner.py" +source = runner.read_text() + +if flag not in source: + import_needle = "from enum import IntEnum\n\nimport torch" + import_replacement = "from enum import IntEnum\n\nimport os\n\nimport torch" + if source.count(import_needle) != 1: + raise RuntimeError(f"Unexpected import layout in {runner}") + source = source.replace(import_needle, import_replacement, 1) + + condition_needle = """ self.enable_k3_latent_moe_tail_fusion = ( + current_platform.is_cuda() + and current_platform.is_device_capability_family(100) + )""" + condition_replacement = f""" self.enable_k3_latent_moe_tail_fusion = ( + current_platform.is_cuda() + and current_platform.is_device_capability_family(100) + and os.getenv("{flag}", "0") != "1" + )""" + if source.count(condition_needle) != 1: + raise RuntimeError(f"Unexpected latent-MoE fusion condition in {runner}") + source = source.replace(condition_needle, condition_replacement, 1) + + temporary = runner.with_suffix(".py.tmp") + temporary.write_text(source) + os.replace(temporary, runner) + +print(f"Prepared portable Kimi K3 latent-MoE path in {runner}") +PY + # The Kimi K3 DSpark checkpoint publishes its parallel-drafting token as # `mask_token_id`. Dynamo's serialized draft config reaches vLLM without the # K3 config-class alias, while vLLM's parallel drafter accepts `pard_token`. diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 355231a6f7..0e1fb26041 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -44,8 +44,9 @@ backend: connector: null aggregated_environment: # K3's fused tail uses torch symmetric memory, which cannot exchange file - # descriptors across this two-node TP group. - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" + # descriptors across this two-node TP group. The compatibility script + # makes this explicit opt-out effective in the pinned vLLM nightly. + INFERENCEX_DISABLE_K3_LATENT_MOE_TAIL_FUSION: "1" # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml index c561167e49..2c109d32cb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml @@ -44,8 +44,9 @@ backend: connector: null aggregated_environment: # K3's fused tail uses torch symmetric memory, which cannot exchange file - # descriptors across this two-node TP group. - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" + # descriptors across this two-node TP group. The compatibility script + # makes this explicit opt-out effective in the pinned vLLM nightly. + INFERENCEX_DISABLE_K3_LATENT_MOE_TAIL_FUSION: "1" # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a38cd30954..4162415675 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5718,4 +5718,5 @@ - "Add a TP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" - "Use the pinned 2026-08-10 vLLM nightly with upstream Kimi K3 and DSpark support" + - "Use the portable PyNCCL latent-MoE path because cross-node B200 cannot rendezvous the SM100 fused tail" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 \ No newline at end of file From eac9a89482036cf1aa0e7045903f43f8e0a73a3a Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:37:50 -0500 Subject: [PATCH 13/32] fix: force portable Kimi collectives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：强制 Kimi 跨节点 latent-MoE 使用可移植的 PyNCCL 通信路径。 --- .../configs/kimik3-dspark-config-compat.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh index 42ea3dadd6..13a1a0b4fd 100755 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh @@ -2,9 +2,9 @@ set -euo pipefail # The pinned vLLM nightly unconditionally enables Kimi K3's SM100 latent-MoE -# tail fusion. That kernel uses torch symmetric memory, whose file-descriptor -# rendezvous is node-local and fails for TP16 across two B200 nodes. Patch the -# pinned implementation to honor the explicit portable-path opt-out below. +# tail and fused all-reduce/RMSNorm collectives. They require symmetric-memory +# transport unavailable to TP16 across two B200 nodes. Patch the pinned +# implementation to honor the explicit portable-path opt-out below. python3 - <<'PY' import importlib.util import os @@ -39,6 +39,17 @@ if flag not in source: raise RuntimeError(f"Unexpected latent-MoE fusion condition in {runner}") source = source.replace(condition_needle, condition_replacement, 1) + fused_norm_needle = ( + " if flashinfer_trtllm_fused_allreduce_norm is not None:" + ) + fused_norm_replacement = f""" if ( + flashinfer_trtllm_fused_allreduce_norm is not None + and os.getenv("{flag}", "0") != "1" + ):""" + if source.count(fused_norm_needle) != 1: + raise RuntimeError(f"Unexpected fused all-reduce condition in {runner}") + source = source.replace(fused_norm_needle, fused_norm_replacement, 1) + temporary = runner.with_suffix(".py.tmp") temporary.write_text(source) os.replace(temporary, runner) From 08ca2385571c3ab6b995cc359370cafb4a31c789 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:47:51 -0500 Subject: [PATCH 14/32] fix: install Kimi compatibility setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：在两个 B200 配方中安装 Kimi 兼容性启动脚本。 --- .../kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 2 ++ .../agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 0e1fb26041..f00f5a5b30 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -17,6 +17,8 @@ identity: dynamo: install: false +setup_script: kimik3-dspark-config-compat.sh + slurm: time_limit: "8:00:00" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml index 2c109d32cb..18d319a8de 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml @@ -17,6 +17,8 @@ identity: dynamo: install: false +setup_script: kimik3-dspark-config-compat.sh + slurm: time_limit: "8:00:00" From 48ea8e6aabc5341229f296d7e488ba068ff71a84 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:01:43 -0500 Subject: [PATCH 15/32] fix: bypass unsupported Kimi workspaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：跳过跨节点 B200 不支持的 Kimi FlashInfer 对称内存工作区探测。 --- .../configs/kimik3-dspark-config-compat.sh | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh index 13a1a0b4fd..525978a535 100755 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh @@ -2,9 +2,9 @@ set -euo pipefail # The pinned vLLM nightly unconditionally enables Kimi K3's SM100 latent-MoE -# tail and fused all-reduce/RMSNorm collectives. They require symmetric-memory -# transport unavailable to TP16 across two B200 nodes. Patch the pinned -# implementation to honor the explicit portable-path opt-out below. +# tail and probes fused all-reduce/RMSNorm workspaces. They require +# symmetric-memory transport unavailable to TP16 across two B200 nodes. Patch +# the pinned implementation to honor the explicit portable-path opt-out below. python3 - <<'PY' import importlib.util import os @@ -54,6 +54,37 @@ if flag not in source: temporary.write_text(source) os.replace(temporary, runner) +communicator = ( + package / "distributed/device_communicators/flashinfer_all_reduce.py" +) +comm_source = communicator.read_text() +if flag not in comm_source: + workspace_needle = """ global _fi_ar_workspace + if _fi_ar_workspace is not None:""" + workspace_replacement = f""" global _fi_ar_workspace + if os.getenv("{flag}", "0") == "1": + return None + if _fi_ar_workspace is not None:""" + if comm_source.count(workspace_needle) != 1: + raise RuntimeError(f"Unexpected all-reduce workspace layout in {communicator}") + comm_source = comm_source.replace( + workspace_needle, workspace_replacement, 1 + ) + + quant_needle = """ global _fi_ar_quant_workspace + if _fi_ar_quant_workspace is not None:""" + quant_replacement = f""" global _fi_ar_quant_workspace + if os.getenv("{flag}", "0") == "1": + return None + if _fi_ar_quant_workspace is not None:""" + if comm_source.count(quant_needle) != 1: + raise RuntimeError(f"Unexpected quant workspace layout in {communicator}") + comm_source = comm_source.replace(quant_needle, quant_replacement, 1) + + temporary = communicator.with_suffix(".py.tmp") + temporary.write_text(comm_source) + os.replace(temporary, communicator) + print(f"Prepared portable Kimi K3 latent-MoE path in {runner}") PY From 6c86afff4e8117c37df60a833f6291c45980c075 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:06:39 -0500 Subject: [PATCH 16/32] fix: force PyNCCL for Kimi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：强制 Kimi 跨节点通信使用 PyNCCL 可移植路径。 --- .../kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 3 ++- .../agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index f00f5a5b30..3b88b7ccfe 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -49,8 +49,9 @@ backend: # descriptors across this two-node TP group. The compatibility script # makes this explicit opt-out effective in the pinned vLLM nightly. INFERENCEX_DISABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + # Force cross-node collectives through the portable PyNCCL path. VLLM_ALLREDUCE_USE_FLASHINFER: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml index 18d319a8de..ad91734f91 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml @@ -49,8 +49,9 @@ backend: # descriptors across this two-node TP group. The compatibility script # makes this explicit opt-out effective in the pinned vLLM nightly. INFERENCEX_DISABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + # Force cross-node collectives through the portable PyNCCL path. VLLM_ALLREDUCE_USE_FLASHINFER: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" From 0d8d71bad08f8cfc7f2c6cba511f7fca27b8e444 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:10:15 -0500 Subject: [PATCH 17/32] fix: terminate Kimi changelog entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：补齐 Kimi 基准测试变更日志条目的末尾换行。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4162415675..c983ae8759 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5719,4 +5719,4 @@ - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" - "Use the pinned 2026-08-10 vLLM nightly with upstream Kimi K3 and DSpark support" - "Use the portable PyNCCL latent-MoE path because cross-node B200 cannot rendezvous the SM100 fused tail" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 \ No newline at end of file + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 From cdcd4a7d317c6006c1681bc504d4e909c4a04813 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:51:31 -0500 Subject: [PATCH 18/32] refactor: remove obsolete DEP accounting changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：移除 TP16 方案不再使用的 DEP 计数逻辑与测试，保持 PR 范围聚焦。 --- .../aggregation/process_agentic_result.py | 37 ++------------ .../test_process_agentic_result.py | 29 ----------- utils/matrix_logic/generate_sweep_configs.py | 39 +++++++-------- .../test_generate_sweep_configs.py | 50 ------------------- 4 files changed, 22 insertions(+), 133 deletions(-) diff --git a/utils/agentic/aggregation/process_agentic_result.py b/utils/agentic/aggregation/process_agentic_result.py index 4971a132ba..5d13fa0f1b 100644 --- a/utils/agentic/aggregation/process_agentic_result.py +++ b/utils/agentic/aggregation/process_agentic_result.py @@ -95,18 +95,6 @@ def _validate_kv_offload_env() -> tuple[str, dict[str, str] | None]: return kv_offloading, backend_metadata -def _parallel_gpu_count( - tp: int, - pp: int, - pcp_size: int, - ep: int, - dp_attention: bool, -) -> int: - """Return the physical GPU count for one worker replica.""" - tensor_gpu_count = tp * pp * pcp_size - return max(tensor_gpu_count, ep) if dp_attention else tensor_gpu_count - - def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: is_multinode = env_bool("IS_MULTINODE") tp = env_int("TP", 1) @@ -123,10 +111,7 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: "PP_SIZE, DCP_SIZE, and PCP_SIZE must be positive integers." ) fields.update({"pp": pp, "dcp_size": dcp_size, "pcp_size": pcp_size}) - num_gpus = _parallel_gpu_count( - tp, pp, pcp_size, ep, env_bool("DP_ATTENTION") - ) - return fields, num_gpus, tp, ep, dp_attention + return fields, tp * pp * pcp_size, tp, ep, dp_attention prefill_num_workers = env_int("PREFILL_NUM_WORKERS") prefill_tp = env_int("PREFILL_TP") @@ -135,7 +120,6 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: prefill_pcp_size = env_int("PREFILL_PCP_SIZE", 1) prefill_ep = env_int("PREFILL_EP", 1) prefill_dp_attention = os.environ.get("PREFILL_DP_ATTN", "false") - prefill_dp_attention_enabled = env_bool("PREFILL_DP_ATTN") decode_num_workers = env_int("DECODE_NUM_WORKERS") decode_tp = env_int("DECODE_TP") decode_pp = env_int("DECODE_PP_SIZE", 1) @@ -143,7 +127,6 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: decode_pcp_size = env_int("DECODE_PCP_SIZE", 1) decode_ep = env_int("DECODE_EP", 1) decode_dp_attention = os.environ.get("DECODE_DP_ATTN", "false") - decode_dp_attention_enabled = env_bool("DECODE_DP_ATTN") worker_parallelism = ( prefill_pp, prefill_dcp_size, @@ -162,20 +145,8 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: raise SystemExit( "PREFILL_HARDWARE and DECODE_HARDWARE must be specified together." ) - num_prefill_gpu = prefill_num_workers * _parallel_gpu_count( - prefill_tp, - prefill_pp, - prefill_pcp_size, - prefill_ep, - prefill_dp_attention_enabled, - ) - num_decode_gpu = decode_num_workers * _parallel_gpu_count( - decode_tp, - decode_pp, - decode_pcp_size, - decode_ep, - decode_dp_attention_enabled, - ) + num_prefill_gpu = prefill_num_workers * prefill_tp * prefill_pp * prefill_pcp_size + num_decode_gpu = decode_num_workers * decode_tp * decode_pp * decode_pcp_size num_gpus = num_prefill_gpu + num_decode_gpu # Aggregated configs set decode num-worker 0 (prefill+decode co-located on one # worker), so there are no separate decode GPUs. Mirror process_result.py and drop @@ -191,7 +162,7 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: ep = max(prefill_ep, decode_ep) dp_attention = ( "true" - if prefill_dp_attention_enabled or decode_dp_attention_enabled + if env_bool("PREFILL_DP_ATTN") or env_bool("DECODE_DP_ATTN") else "false" ) fields.update( diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index 7e98efb1c3..4649c870c0 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -614,35 +614,6 @@ def test_multinode_processor_surfaces_heterogeneous_hardware(tmp_path: Path): agg["num_decode_gpu"], ) == (2, 4, 1, 32) -def test_multinode_processor_counts_attention_dp_ranks( - monkeypatch: pytest.MonkeyPatch, -): - env = { - "IS_MULTINODE": "true", - "PREFILL_NUM_WORKERS": "1", - "PREFILL_TP": "1", - "PREFILL_PP_SIZE": "1", - "PREFILL_DCP_SIZE": "1", - "PREFILL_PCP_SIZE": "1", - "PREFILL_EP": "16", - "PREFILL_DP_ATTN": "true", - "DECODE_NUM_WORKERS": "0", - "DECODE_TP": "1", - "DECODE_PP_SIZE": "1", - "DECODE_DCP_SIZE": "1", - "DECODE_PCP_SIZE": "1", - "DECODE_EP": "1", - "DECODE_DP_ATTN": "false", - } - for name, value in env.items(): - monkeypatch.setenv(name, value) - - fields, num_gpus, tp, ep, dp_attention = _gpu_shape() - - assert fields["num_prefill_gpu"] == 16 - assert fields["num_decode_gpu"] == 0 - assert (num_gpus, tp, ep, dp_attention) == (16, 1, 16, "true") - def test_multinode_processor_omits_homogeneous_hardware(tmp_path: Path): result_dir = _write_fixture(tmp_path) diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 6ccd84b604..86f192b1c8 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -82,21 +82,13 @@ def runner_gpus_per_node(runner: str, runner_data: dict) -> int: return runner_hardware_int(runner, runner_data, Fields.GPUS_PER_NODE.value) -def _parallel_gpu_count(config: dict) -> int: - """Return the GPU footprint encoded by one parallelism configuration.""" - tensor_gpu_count = ( - config[Fields.TP.value] - * config.get(Fields.PP.value, 1) - * config.get(Fields.PCP_SIZE.value, 1) - ) - if config.get(Fields.DP_ATTN.value, False): - return max(tensor_gpu_count, config.get(Fields.EP.value, 1)) - return tensor_gpu_count - - def effective_gpu_count(benchmark: dict) -> int: - """Return GPUs used by a single-node parallel topology.""" - return _parallel_gpu_count(benchmark) + """Return GPUs used by a single-node TP/PP/PCP topology.""" + return ( + benchmark[Fields.TP.value] + * benchmark.get(Fields.PP.value, 1) + * benchmark.get(Fields.PCP_SIZE.value, 1) + ) def with_worker_parallelism_defaults(worker: dict) -> dict: """Return a worker config with explicit parallelism defaults.""" @@ -117,24 +109,29 @@ def worker_gpus_per_node(worker: dict, gpus_per_node: int) -> int: are rejected rather than silently truncated, keeping parity with the single-node "must fit the node" rule: - * A replica larger than one node must fill whole nodes, i.e. be an exact - multiple of gpus-per-node; each of its nodes is then fully occupied. + * A replica larger than one node (tp*pp*pcp > gpus-per-node) must fill whole + nodes, i.e. be an exact multiple of gpus-per-node; each of its nodes is + then fully occupied (fraction 1). * A replica within one node must divide it evenly so co-located replicas of the same role tile the node without overlap. - * Attention-DP topologies include the EP-sized independent rank pool. """ - gpus_per_replica = _parallel_gpu_count(worker) + gpus_per_replica = ( + worker[Fields.TP.value] + * worker.get(Fields.PP.value, 1) + * worker.get(Fields.PCP_SIZE.value, 1) + ) if gpus_per_replica > gpus_per_node: if gpus_per_replica % gpus_per_node != 0: raise ValueError( - f"worker GPU footprint={gpus_per_replica} spans multiple nodes " - "but is not a multiple " + f"worker {Fields.TP.value}*{Fields.PP.value}*{Fields.PCP_SIZE.value}" + f"={gpus_per_replica} spans multiple nodes but is not a multiple " f"of {Fields.GPUS_PER_NODE.value}={gpus_per_node}" ) return gpus_per_node if gpus_per_node % gpus_per_replica != 0: raise ValueError( - f"worker GPU footprint={gpus_per_replica} does not divide " + f"worker {Fields.TP.value}*{Fields.PP.value}*{Fields.PCP_SIZE.value}" + f"={gpus_per_replica} does not divide " f"{Fields.GPUS_PER_NODE.value}={gpus_per_node} evenly" ) return gpus_per_replica diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 47974eb268..7cbfea79af 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2274,56 +2274,6 @@ def test_multinode_agentic_preserves_kv_offload_fields(self, sample_runner_confi # DRAM: 2861022 MiB * 0.80. assert result[0]["total-cpu-dram-gb"] == 2399 - def test_multinode_agentic_dep_budget_counts_attention_dp_ranks( - self, sample_runner_config - ): - config = { - "kimik3-dep": { - "image": "vllm/vllm-openai:kimi-k3", - "model": "moonshotai/Kimi-K3", - "model-prefix": "kimik3", - "precision": "fp4", - "framework": "dynamo-vllm", - "runner": "cluster:b200-dgxc", - "multinode": True, - "scenarios": { - "agentic-coding": [{ - "dram-utilization": 0.61, - "search-space": [{ - "conc-list": [128], - "kv-offloading": "dram", - "kv-offload-backend": {"name": "vllm-simple"}, - "prefill": { - "num-worker": 1, - "tp": 1, - "ep": 16, - "dp-attn": True, - }, - "decode": { - "num-worker": 0, - "tp": 1, - "ep": 1, - "dp-attn": False, - }, - }], - }], - }, - }, - } - args = argparse.Namespace( - config_keys=["kimik3-dep"], - seq_lens=None, - conc=None, - scenario_type=["agentic-coding"], - runner_node_filter=None, - ) - - result = generate_test_config_sweep(args, config, sample_runner_config) - - assert len(result) == 1 - # DEP16 spans both nodes, so all eight ranks on each node share DRAM. - assert result[0]["total-cpu-dram-gb"] == 1829 - def test_multinode_agentic_budget_ignores_decode_topology( self, sample_runner_config ): From c4f8ad5eab0a4b692f88f2ef4e962107cd49624f Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:54:58 -0500 Subject: [PATCH 19/32] fix: reserve sufficient Kimi offload memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将 DRAM 预留比例调整为 0.63，使每节点 1,889 GB 的资源预留覆盖每个 TP rank 220 GiB 的 KV 卸载池。 --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6eb1f878e6..9ae69c1941 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7920,7 +7920,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: disagg: false scenarios: agentic-coding: - - dram-utilization: 0.61 + - dram-utilization: 0.63 search-space: # Low-latency and high-interactivity points. - spec-decoding: mtp From b37142c34441d19edb44e78438e38071410278f5 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:05:52 -0500 Subject: [PATCH 20/32] chore: remove stale DEP launcher comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：移除已不适用于当前 TP16 方案的 DEP 启动器注释。 --- runners/launch_b200-dgxc.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 14d5c60d12..b776d66e58 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -156,8 +156,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then fi # Kimi K3 aggregate profiles use the srt-slurm fork that supports direct - # multi-node vLLM for TP/TEP and topology-derived per-node DP ranks for - # Dynamo DEP. Pin the tested renderer so branch movement cannot change + # multi-node vLLM. Pin the tested renderer so branch movement cannot change # generated rank commands between sweep points. if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then git clone --branch klaud/direct-vllm-multinode --single-branch https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" || exit 1 From 516712809d3c030bf3a558b11e3e60d74665ff3b Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:54:43 -0500 Subject: [PATCH 21/32] fix: cap offload at stable concurrency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：并发 64 预检因 TTFT 指标覆盖率仅为 97.6% 而失败。将 DRAM 卸载容量档上限调整为待验证的并发 48。 --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 9ae69c1941..77201e61cb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7971,7 +7971,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple, version: "13c59a3" } - conc-list: [32, 48, 64] + conc-list: [32, 48] prefill: num-worker: 1 tp: 16 From 629d44704c67f8f144ceba6ef8d601b5733d3569 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:01:04 -0500 Subject: [PATCH 22/32] fix: retain accepted offload capacity endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：并发 64 预检已被工作流接受并上传完整聚合结果，保留该 DRAM 卸载容量端点以刻画饱和区间。 --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 77201e61cb..9ae69c1941 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7971,7 +7971,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple, version: "13c59a3" } - conc-list: [32, 48] + conc-list: [32, 48, 64] prefill: num-worker: 1 tp: 16 From 72b321e3cae88d2587fab3db59065ac66a61d4f0 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:04:44 -0500 Subject: [PATCH 23/32] fix: preserve changelog newline after merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：合并 main 后保留性能变更日志末尾换行 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 32054acbce..dd3692f5b6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5772,4 +5772,4 @@ - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" - "Use the pinned 2026-08-10 vLLM nightly with upstream Kimi K3 and DSpark support" - "Use the portable PyNCCL latent-MoE path because cross-node B200 cannot rendezvous the SM100 fused tail" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 \ No newline at end of file + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 From 977edccd86ac720b4fad1acc5ec541e8d6aa2ef5 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:52:30 -0500 Subject: [PATCH 24/32] fix: use upstream kimi k3 b200 path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：B200 改用上游 TP16+EP16 Kimi K3 推理路径，移除本地 vLLM 补丁，并为 GPU 常驻和 DRAM 卸载配置加入真实 block 验证评估。 --- .../configs/kimik3-dspark-config-compat.sh | 87 -------------- .../agg-b200-tp16-latency-dspark-agentic.yaml | 35 +++--- ...b200-tp16-latency-dspark-eval-agentic.yaml | 110 +++++++++++++++++ ...16-vllm-simple-offload-dspark-agentic.yaml | 35 +++--- ...lm-simple-offload-dspark-eval-agentic.yaml | 111 ++++++++++++++++++ configs/nvidia-master.yaml | 29 +++-- perf-changelog.yaml | 10 +- runners/launch_b200-dgxc.sh | 13 +- 8 files changed, 282 insertions(+), 148 deletions(-) mode change 100755 => 100644 benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh old mode 100755 new mode 100644 index 525978a535..a42c6aba77 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh @@ -1,93 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -# The pinned vLLM nightly unconditionally enables Kimi K3's SM100 latent-MoE -# tail and probes fused all-reduce/RMSNorm workspaces. They require -# symmetric-memory transport unavailable to TP16 across two B200 nodes. Patch -# the pinned implementation to honor the explicit portable-path opt-out below. -python3 - <<'PY' -import importlib.util -import os -from pathlib import Path - -flag = "INFERENCEX_DISABLE_K3_LATENT_MOE_TAIL_FUSION" -spec = importlib.util.find_spec("vllm") -if spec is None or spec.submodule_search_locations is None: - raise RuntimeError("Cannot locate the installed vLLM package") - -package = Path(next(iter(spec.submodule_search_locations))) -runner = package / "models/kimi_k3/nvidia/latent_moe_runner.py" -source = runner.read_text() - -if flag not in source: - import_needle = "from enum import IntEnum\n\nimport torch" - import_replacement = "from enum import IntEnum\n\nimport os\n\nimport torch" - if source.count(import_needle) != 1: - raise RuntimeError(f"Unexpected import layout in {runner}") - source = source.replace(import_needle, import_replacement, 1) - - condition_needle = """ self.enable_k3_latent_moe_tail_fusion = ( - current_platform.is_cuda() - and current_platform.is_device_capability_family(100) - )""" - condition_replacement = f""" self.enable_k3_latent_moe_tail_fusion = ( - current_platform.is_cuda() - and current_platform.is_device_capability_family(100) - and os.getenv("{flag}", "0") != "1" - )""" - if source.count(condition_needle) != 1: - raise RuntimeError(f"Unexpected latent-MoE fusion condition in {runner}") - source = source.replace(condition_needle, condition_replacement, 1) - - fused_norm_needle = ( - " if flashinfer_trtllm_fused_allreduce_norm is not None:" - ) - fused_norm_replacement = f""" if ( - flashinfer_trtllm_fused_allreduce_norm is not None - and os.getenv("{flag}", "0") != "1" - ):""" - if source.count(fused_norm_needle) != 1: - raise RuntimeError(f"Unexpected fused all-reduce condition in {runner}") - source = source.replace(fused_norm_needle, fused_norm_replacement, 1) - - temporary = runner.with_suffix(".py.tmp") - temporary.write_text(source) - os.replace(temporary, runner) - -communicator = ( - package / "distributed/device_communicators/flashinfer_all_reduce.py" -) -comm_source = communicator.read_text() -if flag not in comm_source: - workspace_needle = """ global _fi_ar_workspace - if _fi_ar_workspace is not None:""" - workspace_replacement = f""" global _fi_ar_workspace - if os.getenv("{flag}", "0") == "1": - return None - if _fi_ar_workspace is not None:""" - if comm_source.count(workspace_needle) != 1: - raise RuntimeError(f"Unexpected all-reduce workspace layout in {communicator}") - comm_source = comm_source.replace( - workspace_needle, workspace_replacement, 1 - ) - - quant_needle = """ global _fi_ar_quant_workspace - if _fi_ar_quant_workspace is not None:""" - quant_replacement = f""" global _fi_ar_quant_workspace - if os.getenv("{flag}", "0") == "1": - return None - if _fi_ar_quant_workspace is not None:""" - if comm_source.count(quant_needle) != 1: - raise RuntimeError(f"Unexpected quant workspace layout in {communicator}") - comm_source = comm_source.replace(quant_needle, quant_replacement, 1) - - temporary = communicator.with_suffix(".py.tmp") - temporary.write_text(comm_source) - os.replace(temporary, communicator) - -print(f"Prepared portable Kimi K3 latent-MoE path in {runner}") -PY - # The Kimi K3 DSpark checkpoint publishes its parallel-drafting token as # `mask_token_id`. Dynamo's serialized draft config reaches vLLM without the # K3 config-class alias, while vLLM's parallel drafter accepts `pard_token`. diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 3b88b7ccfe..12d47de279 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -1,8 +1,10 @@ -name: "kimik3-vllm-agg-b200-tp16-latency-dspark-agentic" +name: "kimik3-vllm-agg-b200-tep16-latency-dspark-agentic" -# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. -# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp +# Official balanced multi_node_tep profile for two 8xB200 nodes. TP16 shards +# attention and dense layers across all GPUs while EP16 shards MoE experts, +# keeping vLLM's cross-node-unsupported latent-MoE fused tail out of the +# selected as-shipped code path without modifying the serving engine. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tep model: path: "kimik3" container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" @@ -17,8 +19,6 @@ identity: dynamo: install: false -setup_script: kimik3-dspark-config-compat.sh - slurm: time_limit: "8:00:00" @@ -45,11 +45,8 @@ backend: type: vllm connector: null aggregated_environment: - # K3's fused tail uses torch symmetric memory, which cannot exchange file - # descriptors across this two-node TP group. The compatibility script - # makes this explicit opt-out effective in the pinned vLLM nightly. - INFERENCEX_DISABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - # Force cross-node collectives through the portable PyNCCL path. + # These are shipped vLLM controls for the portable cross-node TP + # collectives used by attention and dense layers on the B200 RDMA fabric. VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" @@ -69,22 +66,20 @@ backend: aggregated: served-model-name: "moonshotai/Kimi-K3" tensor-parallel-size: 16 + enable-expert-parallel: true trust-remote-code: true load-format: fastsafetensors moe-backend: auto - # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can - # OOM at the official 0.95 setting. Preserve the observed-safe headroom. + # Keep headroom for the first FlashInfer MXFP4 MoE workspace. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true - # The generic custom communicator also probes MNNVL and stalls before - # model load on this pool. Use PyNCCL for the cross-node TP group. + # Avoid the custom communicator's MNNVL probe on the cross-node group. disable-custom-all-reduce: true - # The default allreduce/RMS fusion independently allocates a FlashInfer - # MNNVL workspace during profile_run, bypassing the communicator flags. + # Keep all-reduce/RMS fusion on the portable collective path. compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' max-model-len: 1048576 kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' enable-prefix-caching: true enable-auto-tool-choice: true tool-call-parser: kimi_k3 @@ -92,8 +87,8 @@ backend: language-model-only: true max-num-seqs: 8 max-num-batched-tokens: 8192 - # Performance runs use the committed K7 golden synthetic AL. The official - # recipe's real-verification form uses rejection_sample_method=block. + # Throughput runs use the committed K7 golden synthetic AL 3.84. Eval + # selects the paired real-verification recipe instead. speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' sbatch_directives: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml new file mode 100644 index 0000000000..ee15c32e6e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml @@ -0,0 +1,110 @@ +name: "kimik3-vllm-agg-b200-tep16-latency-dspark-eval-agentic" + +# Real-verification eval counterpart to the B200 DSpark throughput recipe. +# It keeps the same official multi_node_tep topology and target settings; only +# DSpark acceptance changes from synthetic golden AL to target-verified block. +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # These are shipped vLLM controls for the portable cross-node TP + # collectives used by attention and dense layers on the B200 RDMA fabric. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Keep headroom for the first FlashInfer MXFP4 MoE workspace. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # Avoid the custom communicator's MNNVL probe on the cross-node group. + disable-custom-all-reduce: true + # Keep all-reduce/RMS fusion on the portable collective path. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Eval must verify every draft against real target logits. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml index ad91734f91..d30d9a2da6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml @@ -1,8 +1,10 @@ -name: "kimik3-vllm-agg-b200-tp16-simple-offload-dspark-agentic" +name: "kimik3-vllm-agg-b200-tep16-simple-offload-dspark-agentic" -# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. -# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp +# Official balanced multi_node_tep profile for two 8xB200 nodes. TP16 shards +# attention and dense layers across all GPUs while EP16 shards MoE experts, +# keeping vLLM's cross-node-unsupported latent-MoE fused tail out of the +# selected as-shipped code path without modifying the serving engine. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tep model: path: "kimik3" container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" @@ -17,8 +19,6 @@ identity: dynamo: install: false -setup_script: kimik3-dspark-config-compat.sh - slurm: time_limit: "8:00:00" @@ -45,11 +45,8 @@ backend: type: vllm connector: null aggregated_environment: - # K3's fused tail uses torch symmetric memory, which cannot exchange file - # descriptors across this two-node TP group. The compatibility script - # makes this explicit opt-out effective in the pinned vLLM nightly. - INFERENCEX_DISABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - # Force cross-node collectives through the portable PyNCCL path. + # These are shipped vLLM controls for the portable cross-node TP + # collectives used by attention and dense layers on the B200 RDMA fabric. VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" @@ -69,22 +66,20 @@ backend: aggregated: served-model-name: "moonshotai/Kimi-K3" tensor-parallel-size: 16 + enable-expert-parallel: true trust-remote-code: true load-format: fastsafetensors moe-backend: auto - # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can - # OOM at the official 0.95 setting. Preserve the observed-safe headroom. + # Keep headroom for the first FlashInfer MXFP4 MoE workspace. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true - # The generic custom communicator also probes MNNVL and stalls before - # model load on this pool. Use PyNCCL for the cross-node TP group. + # Avoid the custom communicator's MNNVL probe on the cross-node group. disable-custom-all-reduce: true - # The default allreduce/RMS fusion independently allocates a FlashInfer - # MNNVL workspace during profile_run, bypassing the communicator flags. + # Keep all-reduce/RMS fusion on the portable collective path. compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' max-model-len: 1048576 kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' enable-prefix-caching: true enable-auto-tool-choice: true tool-call-parser: kimi_k3 @@ -92,8 +87,8 @@ backend: language-model-only: true max-num-seqs: 8 max-num-batched-tokens: 8192 - # Performance runs use the committed K7 golden synthetic AL. The official - # recipe's real-verification form uses rejection_sample_method=block. + # Throughput runs use the committed K7 golden synthetic AL 3.84. Eval + # selects the paired real-verification recipe instead. speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml new file mode 100644 index 0000000000..9e997d1df1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml @@ -0,0 +1,111 @@ +name: "kimik3-vllm-agg-b200-tep16-simple-offload-dspark-eval-agentic" + +# Real-verification eval counterpart to the B200 DSpark offload throughput +# recipe. It keeps the official multi_node_tep topology and CPU KV connector; +# only DSpark acceptance changes from synthetic golden AL to verified block. +model: + path: "kimik3" + container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # These are shipped vLLM controls for the portable cross-node TP + # collectives used by attention and dense layers on the B200 RDMA fabric. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Keep headroom for the first FlashInfer MXFP4 MoE workspace. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # Avoid the custom communicator's MNNVL probe on the cross-node group. + disable-custom-all-reduce: true + # Keep all-reduce/RMS fusion on the portable collective path. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Eval must verify every draft against real target logits. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e350df5e71..c5e3410be4 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8002,11 +8002,10 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false -# DSpark's K3DSparkModel does not implement pipeline parallelism, so Kimi K3 -# DSpark must use the official PP1 TP16 layout across all 16 B200s. Keep plain -# TP because the latency and capacity objectives do not require expert -# parallelism. Split the same serving topology into latency, balanced, -# throughput, and CPU KV-offloaded capacity tiers. +# DSpark does not compose with pipeline parallelism, so use the official +# two-node TP16+EP16 profile. Expert parallel keeps each MoE layer's internal +# TP size at one and therefore avoids the fused latent-MoE symmetric-memory +# path while all 16 GPUs still participate in attention/dense TP. kimik3-fp4-b200-dynamo-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21 model: moonshotai/Kimi-K3 @@ -8026,14 +8025,15 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: prefill: num-worker: 1 tp: 16 - ep: 1 + ep: 16 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml" decode: num-worker: 0 tp: 16 - ep: 1 + ep: 16 dp-attn: false # Balanced medium-concurrency points. - spec-decoding: mtp @@ -8041,14 +8041,15 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: prefill: num-worker: 1 tp: 16 - ep: 1 + ep: 16 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml" decode: num-worker: 0 tp: 16 - ep: 1 + ep: 16 dp-attn: false # GPU-resident throughput points around the prior c16-c32 KV cliff. - spec-decoding: mtp @@ -8056,14 +8057,15 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: prefill: num-worker: 1 tp: 16 - ep: 1 + ep: 16 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml" decode: num-worker: 0 tp: 16 - ep: 1 + ep: 16 dp-attn: false # CPU KV-offloaded capacity points beyond the GPU-resident cliff. - spec-decoding: mtp @@ -8073,14 +8075,15 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: prefill: num-worker: 1 tp: 16 - ep: 1 + ep: 16 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml" decode: num-worker: 0 tp: 16 - ep: 1 + ep: 16 dp-attn: false qwen3.5-fp8-gb300-dynamo-sglang-mtp: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 34ba2fecdb..97c3e28742 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5775,9 +5775,9 @@ - config-keys: - kimik3-fp4-b200-dynamo-vllm-agentic-dspark description: - - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the PP-free TP16 DSpark topology" - - "Add a TP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" - - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" - - "Use the pinned 2026-08-10 vLLM nightly with upstream Kimi K3 and DSpark support" - - "Use the portable PyNCCL latent-MoE path because cross-node B200 cannot rendezvous the SM100 fused tail" + - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the two-node upstream TP16+EP16 DSpark topology" + - "Add a TP16+EP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" + - "Keep throughput on DSpark level 7 probabilistic drafting with committed golden synthetic AL 3.84; EVAL_ONLY selects committed rejection_sample_method=block recipes for real target verification" + - "Use the digest-pinned 2026-08-10 upstream vLLM nightly and direct Inferact/Kimi-K3-DSpark model ID as shipped, with no local compatibility layer or installed serving-engine rewrite" + - "Use vLLM expert parallelism and shipped portable-collective controls to avoid the cross-node latent-MoE symmetric-memory path" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 2ad4265854..8c2b357b00 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -172,11 +172,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then git clone --branch klaud/direct-vllm-multinode --single-branch https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 || exit 1 - mkdir -p recipes/vllm/kimi-k3/agentic configs || exit 1 + mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ recipes/vllm/kimi-k3/agentic || exit 1 - cp "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh" \ - configs/kimik3-dspark-config-compat.sh || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 @@ -349,6 +347,15 @@ EOF echo "MODEL_PATH=$MODEL_PATH (exists=$(test -d "$MODEL_PATH" && echo yes || echo NO))" ls -ld "$MODEL_PATH" 2>&1 || ls /lustre/fsw/models/ 2>&1 | head -40 + # An eval row may point at a committed real-verification recipe while its + # throughput row keeps synthetic golden acceptance. Only configs that set + # EVAL_CONFIG_FILE opt into this selection; all other configs keep using + # CONFIG_FILE unchanged. + if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then + CONFIG_FILE="$EVAL_CONFIG_FILE" + echo "EVAL_ONLY=true: selecting real-verification recipe $CONFIG_FILE" + fi + if [[ -z "$CONFIG_FILE" ]]; then echo "Error: CONFIG_FILE is not set. The srt-slurm path requires a CONFIG_FILE in additional-settings." >&2 echo "Config: MODEL_PREFIX=${MODEL_PREFIX} PRECISION=${PRECISION} FRAMEWORK=${FRAMEWORK}" >&2 From 3be29f2c1c119a6658971cc5fc14a0cd71811b71 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:40:42 -0500 Subject: [PATCH 25/32] fix: run b200 tep in eager mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：B200 双节点 TEP 配置改用上游 eager 模式，避免 CUDA graph capture 探测不可用的跨节点 MNNVL workspace。 --- .../agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 5 +++-- .../agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml | 5 +++-- .../agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml | 5 +++-- ...gg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml | 5 +++-- perf-changelog.yaml | 1 + 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 12d47de279..391bdf1b73 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -75,8 +75,9 @@ backend: no-enable-flashinfer-autotune: true # Avoid the custom communicator's MNNVL probe on the cross-node group. disable-custom-all-reduce: true - # Keep all-reduce/RMS fusion on the portable collective path. - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + # Cross-node B200 has no MNNVL workspace. Upstream eager mode avoids + # tracing the unsupported FlashInfer all-reduce fusion during graph capture. + enforce-eager: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml index ee15c32e6e..0571b037d7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml @@ -73,8 +73,9 @@ backend: no-enable-flashinfer-autotune: true # Avoid the custom communicator's MNNVL probe on the cross-node group. disable-custom-all-reduce: true - # Keep all-reduce/RMS fusion on the portable collective path. - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + # Cross-node B200 has no MNNVL workspace. Upstream eager mode avoids + # tracing the unsupported FlashInfer all-reduce fusion during graph capture. + enforce-eager: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml index d30d9a2da6..98712d417f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml @@ -75,8 +75,9 @@ backend: no-enable-flashinfer-autotune: true # Avoid the custom communicator's MNNVL probe on the cross-node group. disable-custom-all-reduce: true - # Keep all-reduce/RMS fusion on the portable collective path. - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + # Cross-node B200 has no MNNVL workspace. Upstream eager mode avoids + # tracing the unsupported FlashInfer all-reduce fusion during graph capture. + enforce-eager: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml index 9e997d1df1..eb3fca0212 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml @@ -73,8 +73,9 @@ backend: no-enable-flashinfer-autotune: true # Avoid the custom communicator's MNNVL probe on the cross-node group. disable-custom-all-reduce: true - # Keep all-reduce/RMS fusion on the portable collective path. - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + # Cross-node B200 has no MNNVL workspace. Upstream eager mode avoids + # tracing the unsupported FlashInfer all-reduce fusion during graph capture. + enforce-eager: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 97c3e28742..2f5a4d9bdc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5780,4 +5780,5 @@ - "Keep throughput on DSpark level 7 probabilistic drafting with committed golden synthetic AL 3.84; EVAL_ONLY selects committed rejection_sample_method=block recipes for real target verification" - "Use the digest-pinned 2026-08-10 upstream vLLM nightly and direct Inferact/Kimi-K3-DSpark model ID as shipped, with no local compatibility layer or installed serving-engine rewrite" - "Use vLLM expert parallelism and shipped portable-collective controls to avoid the cross-node latent-MoE symmetric-memory path" + - "Run the two-node B200 TEP path in upstream eager mode so graph capture does not probe the unavailable cross-node MNNVL workspace" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 From 7abf1dbf07043ec26711a24540ca4328fb72e287 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:16:15 -0500 Subject: [PATCH 26/32] fix: keep b200 tensor parallel node local MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：B200 双节点拓扑改为节点内 TP8、跨节点 DP2/EP16，使用上游 vLLM 原生路径并避免跨节点 FlashInfer MNNVL workspace。 --- ...g-b200-tp8dp2-latency-dspark-agentic.yaml} | 26 ++++----- ...0-tp8dp2-latency-dspark-eval-agentic.yaml} | 23 +++----- ...2-vllm-simple-offload-dspark-agentic.yaml} | 26 ++++----- ...m-simple-offload-dspark-eval-agentic.yaml} | 23 +++----- configs/nvidia-master.yaml | 55 +++++++++---------- perf-changelog.yaml | 7 +-- 6 files changed, 68 insertions(+), 92 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/{agg-b200-tp16-latency-dspark-agentic.yaml => agg-b200-tp8dp2-latency-dspark-agentic.yaml} (76%) rename benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/{agg-b200-tp16-latency-dspark-eval-agentic.yaml => agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml} (77%) rename benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/{agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml => agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml} (77%) rename benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/{agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml => agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml} (78%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml similarity index 76% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml index 391bdf1b73..41a1ac7ea8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml @@ -1,10 +1,9 @@ -name: "kimik3-vllm-agg-b200-tep16-latency-dspark-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-latency-dspark-agentic" -# Official balanced multi_node_tep profile for two 8xB200 nodes. TP16 shards -# attention and dense layers across all GPUs while EP16 shards MoE experts, -# keeping vLLM's cross-node-unsupported latent-MoE fused tail out of the -# selected as-shipped code path without modifying the serving engine. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tep +# Two-node DEP profile: TP8 stays within each NVSwitch node, DP2 replicates +# attention/dense across nodes, and EP16 shards the 896 experts across all GPUs. +# This keeps FlashInfer fused collectives node-local while using shipped vLLM. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep model: path: "kimik3" container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" @@ -45,10 +44,9 @@ backend: type: vllm connector: null aggregated_environment: - # These are shipped vLLM controls for the portable cross-node TP - # collectives used by attention and dense layers on the B200 RDMA fabric. - VLLM_ALLREDUCE_USE_FLASHINFER: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + # TP8 collectives stay within each B200 NVSwitch domain. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" @@ -65,7 +63,8 @@ backend: vllm_config: aggregated: served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 + tensor-parallel-size: 8 + data-parallel-size: 2 enable-expert-parallel: true trust-remote-code: true load-format: fastsafetensors @@ -73,11 +72,6 @@ backend: # Keep headroom for the first FlashInfer MXFP4 MoE workspace. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true - # Avoid the custom communicator's MNNVL probe on the cross-node group. - disable-custom-all-reduce: true - # Cross-node B200 has no MNNVL workspace. Upstream eager mode avoids - # tracing the unsupported FlashInfer all-reduce fusion during graph capture. - enforce-eager: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml similarity index 77% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml index 0571b037d7..b5f2fa30dc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml @@ -1,8 +1,8 @@ -name: "kimik3-vllm-agg-b200-tep16-latency-dspark-eval-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-latency-dspark-eval-agentic" -# Real-verification eval counterpart to the B200 DSpark throughput recipe. -# It keeps the same official multi_node_tep topology and target settings; only -# DSpark acceptance changes from synthetic golden AL to target-verified block. +# Real-verification eval counterpart to the B200 DEP throughput recipe. +# It keeps TP8/DP2/EP16 and target settings unchanged; only DSpark acceptance +# changes from synthetic golden AL to target-verified block. model: path: "kimik3" container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" @@ -43,10 +43,9 @@ backend: type: vllm connector: null aggregated_environment: - # These are shipped vLLM controls for the portable cross-node TP - # collectives used by attention and dense layers on the B200 RDMA fabric. - VLLM_ALLREDUCE_USE_FLASHINFER: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + # TP8 collectives stay within each B200 NVSwitch domain. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" @@ -63,7 +62,8 @@ backend: vllm_config: aggregated: served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 + tensor-parallel-size: 8 + data-parallel-size: 2 enable-expert-parallel: true trust-remote-code: true load-format: fastsafetensors @@ -71,11 +71,6 @@ backend: # Keep headroom for the first FlashInfer MXFP4 MoE workspace. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true - # Avoid the custom communicator's MNNVL probe on the cross-node group. - disable-custom-all-reduce: true - # Cross-node B200 has no MNNVL workspace. Upstream eager mode avoids - # tracing the unsupported FlashInfer all-reduce fusion during graph capture. - enforce-eager: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml similarity index 77% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml index 98712d417f..a38d0cd930 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml @@ -1,10 +1,9 @@ -name: "kimik3-vllm-agg-b200-tep16-simple-offload-dspark-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-simple-offload-dspark-agentic" -# Official balanced multi_node_tep profile for two 8xB200 nodes. TP16 shards -# attention and dense layers across all GPUs while EP16 shards MoE experts, -# keeping vLLM's cross-node-unsupported latent-MoE fused tail out of the -# selected as-shipped code path without modifying the serving engine. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tep +# Two-node DEP profile: TP8 stays within each NVSwitch node, DP2 replicates +# attention/dense across nodes, and EP16 shards the 896 experts across all GPUs. +# This keeps FlashInfer fused collectives node-local while using shipped vLLM. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep model: path: "kimik3" container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" @@ -45,10 +44,9 @@ backend: type: vllm connector: null aggregated_environment: - # These are shipped vLLM controls for the portable cross-node TP - # collectives used by attention and dense layers on the B200 RDMA fabric. - VLLM_ALLREDUCE_USE_FLASHINFER: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + # TP8 collectives stay within each B200 NVSwitch domain. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" @@ -65,7 +63,8 @@ backend: vllm_config: aggregated: served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 + tensor-parallel-size: 8 + data-parallel-size: 2 enable-expert-parallel: true trust-remote-code: true load-format: fastsafetensors @@ -73,11 +72,6 @@ backend: # Keep headroom for the first FlashInfer MXFP4 MoE workspace. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true - # Avoid the custom communicator's MNNVL probe on the cross-node group. - disable-custom-all-reduce: true - # Cross-node B200 has no MNNVL workspace. Upstream eager mode avoids - # tracing the unsupported FlashInfer all-reduce fusion during graph capture. - enforce-eager: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml similarity index 78% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml index eb3fca0212..0623e602ac 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml @@ -1,8 +1,8 @@ -name: "kimik3-vllm-agg-b200-tep16-simple-offload-dspark-eval-agentic" +name: "kimik3-vllm-agg-b200-tp8dp2-simple-offload-dspark-eval-agentic" -# Real-verification eval counterpart to the B200 DSpark offload throughput -# recipe. It keeps the official multi_node_tep topology and CPU KV connector; -# only DSpark acceptance changes from synthetic golden AL to verified block. +# Real-verification eval counterpart to the B200 DEP offload throughput recipe. +# It keeps TP8/DP2/EP16 and the CPU KV connector unchanged; only DSpark +# acceptance changes from synthetic golden AL to target-verified block. model: path: "kimik3" container: "vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21" @@ -43,10 +43,9 @@ backend: type: vllm connector: null aggregated_environment: - # These are shipped vLLM controls for the portable cross-node TP - # collectives used by attention and dense layers on the B200 RDMA fabric. - VLLM_ALLREDUCE_USE_FLASHINFER: "0" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + # TP8 collectives stay within each B200 NVSwitch domain. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" @@ -63,7 +62,8 @@ backend: vllm_config: aggregated: served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 + tensor-parallel-size: 8 + data-parallel-size: 2 enable-expert-parallel: true trust-remote-code: true load-format: fastsafetensors @@ -71,11 +71,6 @@ backend: # Keep headroom for the first FlashInfer MXFP4 MoE workspace. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true - # Avoid the custom communicator's MNNVL probe on the cross-node group. - disable-custom-all-reduce: true - # Cross-node B200 has no MNNVL workspace. Upstream eager mode avoids - # tracing the unsupported FlashInfer all-reduce fusion during graph capture. - enforce-eager: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c5e3410be4..e904989998 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8002,10 +8002,9 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false -# DSpark does not compose with pipeline parallelism, so use the official -# two-node TP16+EP16 profile. Expert parallel keeps each MoE layer's internal -# TP size at one and therefore avoids the fused latent-MoE symmetric-memory -# path while all 16 GPUs still participate in attention/dense TP. +# DSpark uses two node-local TP8 attention/dense groups and DP2 across nodes. +# Expert parallel spans all 16 ranks, so the 896 experts remain EP16 while +# FlashInfer fused collectives stay inside each B200 NVSwitch domain. kimik3-fp4-b200-dynamo-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21 model: moonshotai/Kimi-K3 @@ -8024,49 +8023,49 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [1, 2, 4] prefill: num-worker: 1 - tp: 16 + tp: 8 ep: 16 - dp-attn: false + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" decode: num-worker: 0 - tp: 16 + tp: 8 ep: 16 - dp-attn: false + dp-attn: true # Balanced medium-concurrency points. - spec-decoding: mtp conc-list: [8, 12, 16] prefill: num-worker: 1 - tp: 16 + tp: 8 ep: 16 - dp-attn: false + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" decode: num-worker: 0 - tp: 16 + tp: 8 ep: 16 - dp-attn: false + dp-attn: true # GPU-resident throughput points around the prior c16-c32 KV cliff. - spec-decoding: mtp conc-list: [20, 24, 28, 32] prefill: num-worker: 1 - tp: 16 + tp: 8 ep: 16 - dp-attn: false + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" decode: num-worker: 0 - tp: 16 + tp: 8 ep: 16 - dp-attn: false + dp-attn: true # CPU KV-offloaded capacity points beyond the GPU-resident cliff. - spec-decoding: mtp kv-offloading: dram @@ -8074,17 +8073,17 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [32, 48, 64] prefill: num-worker: 1 - tp: 16 + tp: 8 ep: 16 - dp-attn: false + dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-eval-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml" + - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml" decode: num-worker: 0 - tp: 16 + tp: 8 ep: 16 - dp-attn: false + dp-attn: true qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2f5a4d9bdc..b57387d6ae 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5775,10 +5775,9 @@ - config-keys: - kimik3-fp4-b200-dynamo-vllm-agentic-dspark description: - - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the two-node upstream TP16+EP16 DSpark topology" - - "Add a TP16+EP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" + - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on a two-node TP8/DP2/EP16 DSpark topology with tensor parallelism local to each B200" + - "Add a TP8/DP2/EP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" - "Keep throughput on DSpark level 7 probabilistic drafting with committed golden synthetic AL 3.84; EVAL_ONLY selects committed rejection_sample_method=block recipes for real target verification" - "Use the digest-pinned 2026-08-10 upstream vLLM nightly and direct Inferact/Kimi-K3-DSpark model ID as shipped, with no local compatibility layer or installed serving-engine rewrite" - - "Use vLLM expert parallelism and shipped portable-collective controls to avoid the cross-node latent-MoE symmetric-memory path" - - "Run the two-node B200 TEP path in upstream eager mode so graph capture does not probe the unavailable cross-node MNNVL workspace" + - "Use node-local TP8 FlashInfer collectives and cross-node DP2/EP16 so the digest-pinned upstream serving engine runs without a compatibility layer or source rewrite" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 From a4764413a11208440b116e1501079c6986916abc Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:59:46 -0500 Subject: [PATCH 27/32] ci: shorten multi-node eval artifact names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：缩短多节点评估产物名称，避免超过 GitHub Actions 的 256 字符上限。 --- .github/workflows/benchmark-multinode-tmpl.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 6b5319c667..0a666a6e90 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -467,7 +467,7 @@ jobs: if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: eval_${{ env.EXP_NAME }}_${{ env.RESULT_FILENAME }} + name: eval_${{ env.RESULT_FILENAME }} path: | meta_env.json results*.json From db438d5e109a5b1a17dd31790988f5c69fabe2eb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 16:36:00 -0500 Subject: [PATCH 28/32] fix: enforce Kimi K3 metrics collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为四个 Kimi K3 配方启用显式 AIPerf 服务端指标端点、提示词缓存明细，并要求导出包含 vllm: 前缀。 --- .../agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml | 3 +++ .../agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml | 3 +++ .../agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml | 3 +++ ...gg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml | 3 +++ 4 files changed, 12 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml index 41a1ac7ea8..7dc3d49793 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml @@ -76,6 +76,7 @@ backend: kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' enable-prefix-caching: true + enable-prompt-tokens-details: true enable-auto-tool-choice: true tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 @@ -94,11 +95,13 @@ srun_options: benchmark: type: custom + aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" IS_MULTINODE: "true" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml index b5f2fa30dc..f562682522 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml @@ -75,6 +75,7 @@ backend: kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' enable-prefix-caching: true + enable-prompt-tokens-details: true enable-auto-tool-choice: true tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 @@ -92,11 +93,13 @@ srun_options: benchmark: type: custom + aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" IS_MULTINODE: "true" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml index a38d0cd930..0119d79219 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml @@ -76,6 +76,7 @@ backend: kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' enable-prefix-caching: true + enable-prompt-tokens-details: true enable-auto-tool-choice: true tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 @@ -95,11 +96,13 @@ srun_options: benchmark: type: custom + aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" IS_MULTINODE: "true" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml index 0623e602ac..a3ae8342b9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml @@ -75,6 +75,7 @@ backend: kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' enable-prefix-caching: true + enable-prompt-tokens-details: true enable-auto-tool-choice: true tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 @@ -93,11 +94,13 @@ srun_options: benchmark: type: custom + aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" IS_MULTINODE: "true" From 2bb6eea6a4298ab8849ba69bd25110b655abe1ce Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 17:46:10 -0500 Subject: [PATCH 29/32] perf: probe Kimi K3 offload crossover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 SimpleCPUOffloadConnector 搜索范围扩展至 c8 至 c64，以便与驻留曲线逐点比较并确定交叉点。 --- configs/nvidia-master.yaml | 5 +++-- perf-changelog.yaml | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7569d42aa8..8b45565c85 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8066,11 +8066,12 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: tp: 8 ep: 16 dp-attn: true - # CPU KV-offloaded capacity points beyond the GPU-resident cliff. + # CPU KV-offload crossover and capacity points. Keep the resident points + # above so the same concurrency can be compared with one variable changed. - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple, version: "13c59a3" } - conc-list: [32, 48, 64] + conc-list: [8, 12, 16, 20, 24, 28, 32, 48, 64] prefill: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 85bfc3d292..8b1f2d3930 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5786,3 +5786,9 @@ - "Use the digest-pinned 2026-08-10 upstream vLLM nightly and direct Inferact/Kimi-K3-DSpark model ID as shipped, with no local compatibility layer or installed serving-engine rewrite" - "Use node-local TP8 FlashInfer collectives and cross-node DP2/EP16 so the digest-pinned upstream serving engine runs without a compatibility layer or source rewrite" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + description: + - "Extend the SimpleCPUOffloadConnector grid to c8/c12/c16/c20/c24/c28/c32/c48/c64 to locate its crossover against the resident curve" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 From 0c06e9badfda5d2ba6c31ab0fcbe8de59cc1e7c6 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 21:54:18 -0500 Subject: [PATCH 30/32] fix: pass Kimi K3 metrics through supported env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移除固定 srt-slurm 不支持的字段，并通过自定义基准环境显式传递聚合 vLLM 指标地址。 --- .../kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml | 2 +- .../agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml | 2 +- .../agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml | 2 +- ...agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml index 7dc3d49793..0c6bb858b2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml @@ -95,12 +95,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml index f562682522..05bf130024 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml @@ -93,12 +93,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml index 0119d79219..97d4d71afa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-agentic.yaml @@ -96,12 +96,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml index a3ae8342b9..0d50ff7ec3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-vllm-simple-offload-dspark-eval-agentic.yaml @@ -94,12 +94,12 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" From 258c6e4f544ceff5fdc8d0007b431d69d17b405f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 21:59:07 -0500 Subject: [PATCH 31/32] chore: document Kimi K3 metrics fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 追加记录固定 srt-slurm 的指标配置修复。 --- perf-changelog.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0bce8d894f..7c036c94ef 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5818,3 +5818,9 @@ description: - "Extend the SimpleCPUOffloadConnector grid to c8/c12/c16/c20/c24/c28/c32/c48/c64 to locate its crossover against the resident curve" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + description: + - "Fix AIPerf server-metrics configuration for the pinned Kimi K3 srt-slurm renderer by passing the aggregate vLLM endpoint through the supported custom-benchmark environment contract" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2569 From 5b132baf606b14e89de647a78df7e07969a7a438 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 01:01:16 -0500 Subject: [PATCH 32/32] perf: retain measured Kimi K3 B200 frontier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 基于完整的十九点 AgentX-fast 扫描，将正式扫描收敛到驻留 KV 的 c2/c4/c8/c12，以及 SimpleCPUOffloadConnector 的 c8/c12/c16/c28/c32/c48。保留低延迟曲线、拐点、紧邻的容量边界和高并发对照，同时移除已被支配的重复点。 --- configs/nvidia-master.yaml | 42 +++++--------------------------------- perf-changelog.yaml | 1 + 2 files changed, 6 insertions(+), 37 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c752b04f67..596bb82050 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8051,9 +8051,9 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: agentic-coding: - dram-utilization: 0.63 search-space: - # Low-latency and high-interactivity points. + # Retained resident latency curve from the completed broad fast sweep. - spec-decoding: mtp - conc-list: [1, 2, 4] + conc-list: [2, 4, 8, 12] prefill: num-worker: 1 tp: 8 @@ -8067,44 +8067,12 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: tp: 8 ep: 16 dp-attn: true - # Balanced medium-concurrency points. - - spec-decoding: mtp - conc-list: [8, 12, 16] - prefill: - num-worker: 1 - tp: 8 - ep: 16 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" - decode: - num-worker: 0 - tp: 8 - ep: 16 - dp-attn: true - # GPU-resident throughput points around the prior c16-c32 KV cliff. - - spec-decoding: mtp - conc-list: [20, 24, 28, 32] - prefill: - num-worker: 1 - tp: 8 - ep: 16 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-agentic.yaml" - - "EVAL_CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8dp2-latency-dspark-eval-agentic.yaml" - decode: - num-worker: 0 - tp: 8 - ep: 16 - dp-attn: true - # CPU KV-offload crossover and capacity points. Keep the resident points - # above so the same concurrency can be compared with one variable changed. + # Retain the measured SimpleCPUOffloadConnector knee, immediate boundary, + # and high-capacity controls from the completed broad fast sweep. - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple, version: "13c59a3" } - conc-list: [8, 12, 16, 20, 24, 28, 32, 48, 64] + conc-list: [8, 12, 16, 28, 32, 48] prefill: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7c036c94ef..7ba903ef45 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5823,4 +5823,5 @@ - kimik3-fp4-b200-dynamo-vllm-agentic-dspark description: - "Fix AIPerf server-metrics configuration for the pinned Kimi K3 srt-slurm renderer by passing the aggregate vLLM endpoint through the supported custom-benchmark environment contract" + - "After the complete 19-point AgentX-fast search, retain resident c2/c4/c8/c12 and SimpleCPUOffloadConnector c8/c12/c16/c28/c32/c48 for the full-duration latency curve, knee, cliff boundary, and high-capacity controls" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2569